]> git.ipfire.org Git - thirdparty/squid.git/blame - src/store.cc
projects / thirdparty / squid.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Removed storeFindFirst and storeFindNext wrappers
[thirdparty/squid.git] / src / store.cc
CommitLineData
ff5731b1 1
30a4f2a8 2/*
c3c5b744 3 * $Id: store.cc,v 1.136 1996/10/24 22:21:30 wessels Exp $
30a4f2a8 4 *
5 * DEBUG: section 20 Storeage Manager
6 * AUTHOR: Harvest Derived
7 *
8 * SQUID Internet Object Cache http://www.nlanr.net/Squid/
9 * --------------------------------------------------------
10 *
11 * Squid is the result of efforts by numerous individuals from the
12 * Internet community. Development is led by Duane Wessels of the
13 * National Laboratory for Applied Network Research and funded by
14 * the National Science Foundation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 *
21 * This program is distributed in the hope that it will be useful,
22 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 * GNU General Public License for more details.
25 *
26 * You should have received a copy of the GNU General Public License
27 * along with this program; if not, write to the Free Software
28 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
29 *
30 */
c943f331 31
32/*
30a4f2a8 33 * Copyright (c) 1994, 1995. All rights reserved.
34 *
35 * The Harvest software was developed by the Internet Research Task
36 * Force Research Group on Resource Discovery (IRTF-RD):
37 *
38 * Mic Bowman of Transarc Corporation.
39 * Peter Danzig of the University of Southern California.
40 * Darren R. Hardy of the University of Colorado at Boulder.
41 * Udi Manber of the University of Arizona.
42 * Michael F. Schwartz of the University of Colorado at Boulder.
43 * Duane Wessels of the University of Colorado at Boulder.
44 *
45 * This copyright notice applies to software in the Harvest
46 * ``src/'' directory only. Users should consult the individual
47 * copyright notices in the ``components/'' subdirectories for
48 * copyright information about other software bundled with the
49 * Harvest source code distribution.
50 *
51 * TERMS OF USE
52 *
53 * The Harvest software may be used and re-distributed without
54 * charge, provided that the software origin and research team are
55 * cited in any use of the system. Most commonly this is
56 * accomplished by including a link to the Harvest Home Page
57 * (http://harvest.cs.colorado.edu/) from the query page of any
58 * Broker you deploy, as well as in the query result pages. These
59 * links are generated automatically by the standard Broker
60 * software distribution.
61 *
62 * The Harvest software is provided ``as is'', without express or
63 * implied warranty, and with no support nor obligation to assist
64 * in its use, correction, modification or enhancement. We assume
65 * no liability with respect to the infringement of copyrights,
66 * trade secrets, or any patents, and are not responsible for
67 * consequential damages. Proper use of the Harvest software is
68 * entirely the responsibility of the user.
69 *
70 * DERIVATIVE WORKS
71 *
72 * Users may make derivative works from the Harvest software, subject
73 * to the following constraints:
74 *
75 * - You must include the above copyright notice and these
76 * accompanying paragraphs in all forms of derivative works,
77 * and any documentation and other materials related to such
78 * distribution and use acknowledge that the software was
79 * developed at the above institutions.
80 *
81 * - You must notify IRTF-RD regarding your distribution of
82 * the derivative work.
83 *
84 * - You must clearly notify users that your are distributing
85 * a modified version and not the original Harvest software.
86 *
87 * - Any derivative product is also subject to these copyright
88 * and use restrictions.
89 *
90 * Note that the Harvest software is NOT in the public domain. We
91 * retain copyright, as specified above.
92 *
93 * HISTORY OF FREE SOFTWARE STATUS
94 *
95 * Originally we required sites to license the software in cases
96 * where they were going to build commercial products/services
97 * around Harvest. In June 1995 we changed this policy. We now
98 * allow people to use the core Harvest software (the code found in
99 * the Harvest ``src/'' directory) for free. We made this change
100 * in the interest of encouraging the widest possible deployment of
101 * the technology. The Harvest software is really a reference
102 * implementation of a set of protocols and formats, some of which
103 * we intend to standardize. We encourage commercial
104 * re-implementations of code complying to this set of standards.
c943f331 105 */
090089c4 106
107/*
108 * Here is a summary of the routines which change mem_status and swap_status:
109 * Added 11/18/95
110 *
111 * Routine mem_status swap_status status
112 * ---------------------------------------------------------------------------
6eb42cae 113 * storeCreateEntry NOT_IN_MEMORY NO_SWAP
090089c4 114 * storeComplete IN_MEMORY NO_SWAP
115 * storeSwapOutStart SWAPPING_OUT
116 * storeSwapOutHandle(fail) NO_SWAP
117 * storeSwapOutHandle(ok) SWAP_OK
118 * ---------------------------------------------------------------------------
119 * storeAddDiskRestore NOT_IN_MEMORY SWAP_OK
120 * storeSwapInStart SWAPPING_IN
121 * storeSwapInHandle(fail) NOT_IN_MEMORY
122 * storeSwapInHandle(ok) IN_MEMORY
123 * ---------------------------------------------------------------------------
124 * storeAbort IN_MEMORY NO_SWAP
125 * storePurgeMem NOT_IN_MEMORY
126 * ---------------------------------------------------------------------------
127 * You can reclaim an object's space if it's:
128 * storeGetSwapSpace !SWAPPING_IN !SWAPPING_OUT !STORE_PENDING
129 *
130 */
131
44a47c6e 132#include "squid.h" /* goes first */
090089c4 133
134#define REBUILD_TIMESTAMP_DELTA_MAX 2
135#define MAX_SWAP_FILE (1<<21)
136#define SWAP_BUF DISK_PAGE_SIZE
090089c4 137
227fbb74 138#define WITH_MEMOBJ 1
139#define WITHOUT_MEMOBJ 0
140
090089c4 141/* rate of checking expired objects in main loop */
30a4f2a8 142#define STORE_MAINTAIN_RATE (10)
090089c4 143
144#define STORE_BUCKETS (7921)
fc07a0e4 145#define STORE_IN_MEM_BUCKETS (143)
090089c4 146
2285407f 147#define STORE_LOG_CREATE 0
148#define STORE_LOG_SWAPIN 1
149#define STORE_LOG_SWAPOUT 2
150#define STORE_LOG_RELEASE 3
151
147d3115 152static char *storeLogTags[] =
153{
154 "CREATE",
155 "SWAPIN",
156 "SWAPOUT",
157 "RELEASE"
2285407f 158};
159
e62d2dea 160char *memStatusStr[] =
161{
9dfb6c1c 162 "NOT_IN_MEMORY",
163 "SWAPPING_IN",
164 "IN_MEMORY"
165};
166
e62d2dea 167char *pingStatusStr[] =
168{
9dfb6c1c 169 "PING_WAITING",
170 "PING_TIMEOUT",
171 "PING_DONE",
172 "PING_NONE"
173};
174
e62d2dea 175char *storeStatusStr[] =
176{
9dfb6c1c 177 "STORE_OK",
178 "STORE_PENDING",
179 "STORE_ABORTED"
180};
181
e62d2dea 182char *swapStatusStr[] =
183{
9dfb6c1c 184 "NO_SWAP",
185 "SWAPPING_OUT",
186 "SWAP_OK"
187};
188
3facc31e 189struct storeRebuild_data {
190 FILE *log;
191 int objcount; /* # objects successfully reloaded */
192 int expcount; /* # objects expired */
193 int linecount; /* # lines parsed from cache logfile */
194 int clashcount; /* # swapfile clashes avoided */
195 int dupcount; /* # duplicates purged */
196 time_t start, stop;
30a4f2a8 197 int speed; /* # Objects per run */
3facc31e 198 char line_in[4096];
199};
200
30a4f2a8 201/* initializtion flag */
202int store_rebuilding = STORE_REBUILDING_SLOW;
203
204/* Static Functions */
24382924 205static char *storeDescribeStatus _PARAMS((StoreEntry *));
206static char *storeSwapFullPath _PARAMS((int, char *));
207static HashID storeCreateHashTable _PARAMS((int (*)_PARAMS((char *, char *))));
24382924 208static int compareLastRef _PARAMS((StoreEntry **, StoreEntry **));
209static int compareSize _PARAMS((StoreEntry **, StoreEntry **));
210static int storeAddSwapDisk _PARAMS((char *));
211static int storeCheckExpired _PARAMS((StoreEntry *));
212static int storeCheckPurgeMem _PARAMS((StoreEntry *));
213static int storeClientListSearch _PARAMS((MemObject *, int));
214static int storeCopy _PARAMS((StoreEntry *, int, int, char *, int *));
215static int storeEntryLocked _PARAMS((StoreEntry *));
216static int storeEntryValidLength _PARAMS((StoreEntry *));
217static int storeGetMemSpace _PARAMS((int));
218static int storeHashDelete _PARAMS((StoreEntry *));
219static int storeShouldPurgeMem _PARAMS((StoreEntry *));
220static int storeSwapInHandle _PARAMS((int, char *, int, int, StoreEntry *, int));
67508012 221static int storeSwapInStart _PARAMS((StoreEntry *, SIH, void *));
24382924 222static int swapInError _PARAMS((int, StoreEntry *));
223static mem_ptr new_MemObjectData _PARAMS((void));
224static MemObject *new_MemObject _PARAMS((void));
225static StoreEntry *new_StoreEntry _PARAMS((int));
226static StoreEntry *storeAddDiskRestore _PARAMS((char *, int, int, time_t, time_t, time_t));
227static StoreEntry *storeGetInMemFirst _PARAMS((void));
228static StoreEntry *storeGetInMemNext _PARAMS((void));
229static unsigned int storeGetBucketNum _PARAMS((void));
67508012 230static void destroy_MemObject _PARAMS((MemObject *));
231static void destroy_MemObjectData _PARAMS((MemObject *));
232static void destroy_StoreEntry _PARAMS((StoreEntry *));
24382924 233static void storeClientListAdd _PARAMS((StoreEntry *, int, int));
234static void storeDeleteBehind _PARAMS((StoreEntry *));
235static void storePurgeMem _PARAMS((StoreEntry *));
236static void storeSanityCheck _PARAMS((void));
237static void storeSetMemStatus _PARAMS((StoreEntry *, mem_status_t));
238static void storeStartRebuildFromDisk _PARAMS((void));
67508012 239static void storeSwapLog _PARAMS((StoreEntry *));
24382924 240static void storeSwapOutHandle _PARAMS((int, int, StoreEntry *));
67508012 241static void storeHashMemInsert _PARAMS((StoreEntry *));
242static void storeHashMemDelete _PARAMS((StoreEntry *));
30a4f2a8 243
090089c4 244/* Now, this table is inaccessible to outsider. They have to use a method
245 * to access a value in internal storage data structure. */
24382924 246static HashID store_table = 0;
090089c4 247/* hash table for in-memory-only objects */
24382924 248static HashID in_mem_table = 0;
090089c4 249
090089c4 250/* current memory storage size */
e954773d 251unsigned long store_mem_size = 0;
252
253static int store_pages_high = 0;
254static int store_pages_low = 0;
090089c4 255
090089c4 256/* current file name, swap file, use number as a filename */
cb0486c3 257static int swapfileno = 0;
090089c4 258static int store_swap_size = 0; /* kilobytes !! */
259static unsigned long store_swap_high = 0;
260static unsigned long store_swap_low = 0;
2285407f 261static int swaplog_fd = -1;
262static int swaplog_lock = 0;
2285407f 263static int storelog_fd = -1;
090089c4 264
090089c4 265/* key temp buffer */
ec74cf5f 266static char key_temp_buffer[MAX_URL + 100];
c943f331 267static char swaplog_file[MAX_FILE_NAME_LEN];
3facc31e 268static char tmp_filename[MAX_FILE_NAME_LEN];
090089c4 269
270/* patch cache_dir to accomodate multiple disk storage */
24382924 271static char **CacheDirs = NULL;
a1b0d7cf 272static int CacheDirsAllocated = 0;
090089c4 273int ncache_dirs = 0;
274
b8d8561b 275static MemObject *
276new_MemObject(void)
227fbb74 277{
30a4f2a8 278 MemObject *mem = get_free_mem_obj();
279 mem->reply = xcalloc(1, sizeof(struct _http_reply));
579a45cd 280 meta_data.mem_obj_count++;
30a4f2a8 281 meta_data.misc += sizeof(struct _http_reply);
282 debug(20, 3, "new_MemObject: returning %p\n", mem);
283 return mem;
227fbb74 284}
285
b8d8561b 286static StoreEntry *
287new_StoreEntry(int mem_obj_flag)
090089c4 288{
289 StoreEntry *e = NULL;
290
30a4f2a8 291 e = xcalloc(1, sizeof(StoreEntry));
227fbb74 292 meta_data.store_entries++;
293 if (mem_obj_flag)
294 e->mem_obj = new_MemObject();
295 debug(20, 3, "new_StoreEntry: returning %p\n", e);
296 return e;
090089c4 297}
298
b8d8561b 299static void
300destroy_MemObject(MemObject * mem)
090089c4 301{
30a4f2a8 302 int i;
303 debug(20, 3, "destroy_MemObject: destroying %p\n", mem);
304 destroy_MemObjectData(mem);
305 safe_free(mem->pending);
306 if (mem->client_list) {
6e40f263 307 for (i = 0; i < mem->client_list_size; i++)
308 safe_free(mem->client_list[i]);
30a4f2a8 309 safe_free(mem->client_list);
310 }
311 safe_free(mem->mime_hdr);
312 safe_free(mem->reply);
313 safe_free(mem->e_abort_msg);
314 requestUnlink(mem->request);
315 mem->request = NULL;
1061b406 316 memset(mem, '\0', sizeof(MemObject));
30a4f2a8 317 put_free_mem_obj(mem);
579a45cd 318 meta_data.mem_obj_count--;
30a4f2a8 319 meta_data.misc -= sizeof(struct _http_reply);
090089c4 320}
321
b8d8561b 322static void
323destroy_StoreEntry(StoreEntry * e)
090089c4 324{
227fbb74 325 debug(20, 3, "destroy_StoreEntry: destroying %p\n", e);
85b701ed 326 if (!e) {
327 debug_trap("destroy_StoreEntry: NULL Entry");
328 return;
329 }
227fbb74 330 if (e->mem_obj)
331 destroy_MemObject(e->mem_obj);
30a4f2a8 332 if (e->url) {
333 meta_data.url_strings -= strlen(e->url);
334 safe_free(e->url);
335 } else {
336 debug(20, 3, "destroy_StoreEntry: WARNING! Entry without URL string!\n");
337 }
338 if (BIT_TEST(e->flag, KEY_URL))
339 e->key = NULL;
340 else
341 safe_free(e->key);
1061b406 342 memset(e, '\0', sizeof(StoreEntry));
227fbb74 343 xfree(e);
344 meta_data.store_entries--;
090089c4 345}
346
b8d8561b 347static mem_ptr
348new_MemObjectData(void)
227fbb74 349{
350 debug(20, 3, "new_MemObjectData: calling memInit()\n");
579a45cd 351 meta_data.mem_data_count++;
227fbb74 352 return memInit();
353}
090089c4 354
b8d8561b 355static void
356destroy_MemObjectData(MemObject * mem)
090089c4 357{
e954773d 358 debug(20, 3, "destroy_MemObjectData: destroying %p, %d bytes\n",
359 mem->data, mem->e_current_len);
30a4f2a8 360 store_mem_size -= mem->e_current_len - mem->e_lowest_offset;
30a4f2a8 361 if (mem->data) {
362 mem->data->mem_free(mem->data);
363 mem->data = NULL;
579a45cd 364 meta_data.mem_data_count--;
090089c4 365 }
30a4f2a8 366 mem->e_current_len = 0;
090089c4 367}
368
090089c4 369/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
370
371/*
372 * Create 2 hash tables, "table" has all objects, "in_mem_table" has only
373 * objects in the memory.
374 */
375
24382924 376static HashID
b8d8561b 377storeCreateHashTable(int (*cmp_func) (char *, char *))
090089c4 378{
ec74cf5f 379 store_table = hash_create(cmp_func, STORE_BUCKETS, hash_url);
380 in_mem_table = hash_create(cmp_func, STORE_IN_MEM_BUCKETS, hash_url);
381 return store_table;
090089c4 382}
383
b8d8561b 384static void
385storeHashMemInsert(StoreEntry * e)
b32508fb 386{
387 hash_insert(in_mem_table, e->key, e);
388 meta_data.hot_vm++;
389}
390
b8d8561b 391static void
392storeHashMemDelete(StoreEntry * e)
b32508fb 393{
394 hash_link *hptr = hash_lookup(in_mem_table, e->key);
85b701ed 395 if (hptr == NULL) {
396 debug_trap("storeHashMemDelete: key not found");
397 return;
398 }
b32508fb 399 hash_delete_link(in_mem_table, hptr);
400 meta_data.hot_vm--;
401}
090089c4 402
b8d8561b 403static int
404storeHashInsert(StoreEntry * e)
090089c4 405{
227fbb74 406 debug(20, 3, "storeHashInsert: Inserting Entry %p key '%s'\n",
407 e, e->key);
090089c4 408 if (e->mem_status == IN_MEMORY)
b32508fb 409 storeHashMemInsert(e);
410 return hash_join(store_table, (hash_link *) e);
090089c4 411}
412
b8d8561b 413static int
414storeHashDelete(StoreEntry * e)
090089c4 415{
b32508fb 416 if (e->mem_status == IN_MEMORY)
417 storeHashMemDelete(e);
418 return hash_remove_link(store_table, (hash_link *) e);
090089c4 419}
420
421/*
422 * maintain the in-mem hash table according to the changes of mem_status
234967c9 423 * This routine replaces the instruction "e->store_status = status;"
090089c4 424 */
24382924 425static void
b8d8561b 426storeSetMemStatus(StoreEntry * e, mem_status_t status)
090089c4 427{
85b701ed 428 if (e->key == NULL) {
429 debug_trap("storeSetMemStatus: NULL key");
430 return;
431 } else if (status != IN_MEMORY && e->mem_status == IN_MEMORY)
b32508fb 432 storeHashMemDelete(e);
433 else if (status == IN_MEMORY && e->mem_status != IN_MEMORY)
434 storeHashMemInsert(e);
090089c4 435 e->mem_status = status;
436}
437
438/* -------------------------------------------------------------------------- */
439
b8d8561b 440static char *
441time_describe(time_t t)
2285407f 442{
95d659f0 443 LOCAL_ARRAY(char, buf, 128);
2285407f 444
147d3115 445 if (t < 60) {
446 sprintf(buf, "%ds", (int) t);
447 } else if (t < 3600) {
448 sprintf(buf, "%dm", (int) t / 60);
449 } else if (t < 86400) {
450 sprintf(buf, "%dh", (int) t / 3600);
451 } else if (t < 604800) {
452 sprintf(buf, "%dD", (int) t / 86400);
453 } else if (t < 2592000) {
454 sprintf(buf, "%dW", (int) t / 604800);
455 } else if (t < 31536000) {
456 sprintf(buf, "%dM", (int) t / 2592000);
457 } else {
458 sprintf(buf, "%dY", (int) t / 31536000);
459 }
460 return buf;
2285407f 461}
462
b8d8561b 463static void
464storeLog(int tag, StoreEntry * e)
2285407f 465{
0be3ec60 466 LOCAL_ARRAY(char, logmsg, MAX_URL << 1);
2285407f 467 time_t t;
468 int expect_len = 0;
469 int actual_len = 0;
470 int code = 0;
2285407f 471 if (storelog_fd < 0)
472 return;
b8de7ebe 473 t = e->expires - squid_curtime;
2285407f 474 if (e->mem_obj) {
475 code = e->mem_obj->reply->code;
476 expect_len = (int) e->mem_obj->reply->content_length;
477 actual_len = (int) e->mem_obj->e_current_len - e->mem_obj->reply->hdr_sz;
478 }
479 sprintf(logmsg, "%9d.%03d %-7s %4d %9d [%3s] %d/%d %s\n",
480 (int) current_time.tv_sec,
481 (int) current_time.tv_usec / 1000,
482 storeLogTags[tag],
483 code,
484 (int) t,
485 time_describe(t),
486 expect_len,
487 actual_len,
488 e->key);
489 file_write(storelog_fd,
490 xstrdup(logmsg),
491 strlen(logmsg),
492 0,
493 NULL,
e47afcdb 494 NULL,
495 xfree);
2285407f 496}
497
090089c4 498
499/* get rid of memory copy of the object */
620da955 500/* Only call this if storeCheckPurgeMem(e) returns 1 */
24382924 501static void
b8d8561b 502storePurgeMem(StoreEntry * e)
090089c4 503{
2285407f 504 debug(20, 3, "storePurgeMem: Freeing memory-copy of %s\n", e->key);
2aca8433 505 if (e->mem_obj == NULL)
090089c4 506 return;
090089c4 507 storeSetMemStatus(e, NOT_IN_MEMORY);
227fbb74 508 destroy_MemObject(e->mem_obj);
509 e->mem_obj = NULL;
090089c4 510}
511
512/* lock the object for reading, start swapping in if necessary */
30a4f2a8 513/* Called by:
620da955 514 * icpProcessRequest()
30a4f2a8 515 * storeAbort()
516 * {http,ftp,gopher,wais}Start()
517 */
b8d8561b 518int
519storeLockObject(StoreEntry * e, SIH handler, void *data)
090089c4 520{
090089c4 521 int status = 0;
090089c4 522 e->lock_count++;
148a2921 523 debug(20, 3, "storeLockObject: key '%s' count=%d\n",
524 e->key, (int) e->lock_count);
a1e47288 525 if (e->mem_status != NOT_IN_MEMORY)
526 /* ok, its either IN_MEMORY or SWAPPING_IN */
6e40f263 527 debug(20, 5, "storeLockObject: OK: mem_status is %s\n", memStatusStr[e->mem_status]);
a1e47288 528 else if (e->swap_status == SWAP_OK)
529 /* ok, its NOT_IN_MEMORY, but its swapped out */
6e40f263 530 debug(20, 5, "storeLockObject: OK: swap_status is %s\n", swapStatusStr[e->swap_status]);
a1e47288 531 else if (e->store_status == STORE_PENDING)
532 /* ok, we're reading it in right now */
6e40f263 533 debug(20, 5, "storeLockObject: OK: store_status is %s\n", storeStatusStr[e->store_status]);
534 else
a1e47288 535 fatal_dump(storeDescribeStatus(e));
b8de7ebe 536 e->lastref = squid_curtime;
30a4f2a8 537 /* If the object is NOT_IN_MEMORY, fault it in. */
090089c4 538 if ((e->mem_status == NOT_IN_MEMORY) && (e->swap_status == SWAP_OK)) {
539 /* object is in disk and no swapping daemon running. Bring it in. */
e6f18b4a 540 if ((status = storeSwapInStart(e, handler, data)) < 0) {
541 /* We couldn't find or couldn't open object's swapfile.
090089c4 542 * So, return a -1 here, indicating that we will treat
543 * the reference like a MISS_TTL, force a keychange and
544 storeRelease. */
545 e->lock_count--;
546 }
30a4f2a8 547 } else if (e->mem_status == IN_MEMORY && handler) {
548 /* its already in memory, so call the handler */
e6f18b4a 549 handler(0, data);
30a4f2a8 550 } else if (handler) {
551 /* The object is probably in state SWAPPING_IN, not much we can do.
552 * Instead of returning failure here, we should have a list of complete
553 * handlers which we could append to... */
e6f18b4a 554 handler(1, data);
090089c4 555 }
556 return status;
557}
558
b8d8561b 559void
560storeReleaseRequest(StoreEntry * e)
2285407f 561{
aadcfbad 562 if (BIT_TEST(e->flag, RELEASE_REQUEST))
58bcd31b 563 return;
85b701ed 564 if (!storeEntryLocked(e)) {
aadcfbad 565 debug_trap("Someone called storeReleaseRequest on an unlocked entry");
97992d17 566 debug(20,0," --> '%s'\n", e->url ? e->url : "NULL URL");
85b701ed 567 return;
568 }
2daae136 569 debug(20, 3, "storeReleaseRequest: FOR '%s'\n", e->key ? e->key : e->url);
d3c1a245 570 e->flag |= RELEASE_REQUEST;
2285407f 571}
572
090089c4 573/* unlock object, return -1 if object get released after unlock
574 * otherwise lock_count */
b8d8561b 575int
576storeUnlockObject(StoreEntry * e)
090089c4 577{
6c895381 578 MemObject *mem = e->mem_obj;
579 e->lock_count--;
580 debug(20, 3, "storeUnlockObject: key '%s' count=%d\n",
581 e->key, e->lock_count);
30a4f2a8 582 if (e->lock_count)
a1e47288 583 return (int) e->lock_count;
5b00be7a 584 if (e->store_status == STORE_PENDING) {
585 debug_trap("storeUnlockObject: Someone unlocked STORE_PENDING object");
9d90e665 586 e->store_status = STORE_ABORTED;
5b00be7a 587 }
aadcfbad 588 if (BIT_TEST(e->flag, RELEASE_REQUEST)) {
30a4f2a8 589 storeRelease(e);
aadcfbad 590 } else if (BIT_TEST(e->flag, ABORT_MSG_PENDING)) {
30a4f2a8 591 /* This is where the negative cache gets storeAppended */
592 /* Briefly lock to replace content with abort message */
593 e->lock_count++;
6c895381 594 destroy_MemObjectData(mem);
30a4f2a8 595 e->object_len = 0;
6c895381 596 mem->data = new_MemObjectData();
597 storeAppend(e, mem->e_abort_msg, strlen(mem->e_abort_msg));
598 e->object_len = mem->e_current_len = strlen(mem->e_abort_msg);
30a4f2a8 599 BIT_RESET(e->flag, ABORT_MSG_PENDING);
600 e->lock_count--;
56f29785 601 } else if (storeShouldPurgeMem(e)) {
30a4f2a8 602 storePurgeMem(e);
090089c4 603 }
6c895381 604 return 0;
090089c4 605}
606
607/* Lookup an object in the cache.
608 * return just a reference to object, don't start swapping in yet. */
b8d8561b 609StoreEntry *
610storeGet(char *url)
090089c4 611{
6602e70e 612 debug(20, 3, "storeGet: looking up %s\n", url);
b15fe823 613 return (StoreEntry *) hash_lookup(store_table, url);
090089c4 614}
615
b8d8561b 616unsigned int
617getKeyCounter(void)
04e8dbaa 618{
2d25dc1a 619 static unsigned int key_counter = 0;
620 if (++key_counter == 0)
621 ++key_counter;
622 return key_counter;
04e8dbaa 623}
624
b8d8561b 625char *
626storeGeneratePrivateKey(char *url, method_t method, int num)
6eb42cae 627{
6eb42cae 628 if (num == 0)
04e8dbaa 629 num = getKeyCounter();
6eb42cae 630 debug(20, 3, "storeGeneratePrivateKey: '%s'\n", url);
631 key_temp_buffer[0] = '\0';
632 sprintf(key_temp_buffer, "%d/%s/%s",
633 num,
73ddce89 634 RequestMethodStr[method],
6eb42cae 635 url);
636 return key_temp_buffer;
637}
638
b8d8561b 639char *
640storeGeneratePublicKey(char *url, method_t method)
090089c4 641{
73ddce89 642 debug(20, 3, "storeGeneratePublicKey: type=%d %s\n", method, url);
643 switch (method) {
6eb42cae 644 case METHOD_GET:
227fbb74 645 return url;
983061ed 646 /* NOTREACHED */
227fbb74 647 break;
6eb42cae 648 case METHOD_POST:
090089c4 649 sprintf(key_temp_buffer, "/post/%s", url);
650 return key_temp_buffer;
983061ed 651 /* NOTREACHED */
227fbb74 652 break;
30a4f2a8 653 case METHOD_PUT:
654 sprintf(key_temp_buffer, "/put/%s", url);
655 return key_temp_buffer;
656 /* NOTREACHED */
657 break;
6eb42cae 658 case METHOD_HEAD:
090089c4 659 sprintf(key_temp_buffer, "/head/%s", url);
660 return key_temp_buffer;
983061ed 661 /* NOTREACHED */
227fbb74 662 break;
3facc31e 663 case METHOD_CONNECT:
664 sprintf(key_temp_buffer, "/connect/%s", url);
665 return key_temp_buffer;
983061ed 666 /* NOTREACHED */
3facc31e 667 break;
227fbb74 668 default:
85b701ed 669 debug_trap("storeGeneratePublicKey: Unsupported request method");
227fbb74 670 break;
671 }
672 return NULL;
673}
674
b8d8561b 675void
676storeSetPrivateKey(StoreEntry * e)
227fbb74 677{
6eb42cae 678 StoreEntry *e2 = NULL;
679 hash_link *table_entry = NULL;
07622ccd 680 char *newkey = NULL;
227fbb74 681
6eb42cae 682 if (e->key && BIT_TEST(e->flag, KEY_PRIVATE))
683 return; /* is already private */
090089c4 684
73ddce89 685 newkey = storeGeneratePrivateKey(e->url, e->method, 0);
ec74cf5f 686 if ((table_entry = hash_lookup(store_table, newkey))) {
07622ccd 687 e2 = (StoreEntry *) table_entry;
688 debug(20, 0, "storeSetPrivateKey: Entry already exists with key '%s'\n",
689 newkey);
690 debug(20, 0, "storeSetPrivateKey: Entry Dump:\n%s\n", storeToString(e2));
85b701ed 691 debug_trap("Private key already exists.");
692 return;
227fbb74 693 }
07622ccd 694 if (e->key)
edae7bc0 695 storeHashDelete(e);
07622ccd 696 if (e->key && !BIT_TEST(e->flag, KEY_URL))
697 safe_free(e->key);
698 e->key = xstrdup(newkey);
6eb42cae 699 storeHashInsert(e);
227fbb74 700 BIT_RESET(e->flag, KEY_URL);
6eb42cae 701 BIT_SET(e->flag, KEY_CHANGE);
702 BIT_SET(e->flag, KEY_PRIVATE);
227fbb74 703}
704
b8d8561b 705void
706storeSetPublicKey(StoreEntry * e)
227fbb74 707{
6eb42cae 708 StoreEntry *e2 = NULL;
709 hash_link *table_entry = NULL;
07622ccd 710 char *newkey = NULL;
234967c9 711 int loop_detect = 0;
6eb42cae 712
713 if (e->key && !BIT_TEST(e->flag, KEY_PRIVATE))
714 return; /* is already public */
715
73ddce89 716 newkey = storeGeneratePublicKey(e->url, e->method);
ec74cf5f 717 while ((table_entry = hash_lookup(store_table, newkey))) {
30a4f2a8 718 debug(20, 3, "storeSetPublicKey: Making old '%s' private.\n", newkey);
07622ccd 719 e2 = (StoreEntry *) table_entry;
720 storeSetPrivateKey(e2);
30a4f2a8 721 storeRelease(e2);
234967c9 722 if (loop_detect++ == 10)
723 fatal_dump("storeSetPublicKey() is looping!!");
724 newkey = storeGeneratePublicKey(e->url, e->method);
6eb42cae 725 }
07622ccd 726 if (e->key)
edae7bc0 727 storeHashDelete(e);
07622ccd 728 if (e->key && !BIT_TEST(e->flag, KEY_URL))
729 safe_free(e->key);
73ddce89 730 if (e->method == METHOD_GET) {
6eb42cae 731 e->key = e->url;
732 BIT_SET(e->flag, KEY_URL);
733 BIT_RESET(e->flag, KEY_CHANGE);
734 } else {
07622ccd 735 e->key = xstrdup(newkey);
6eb42cae 736 BIT_RESET(e->flag, KEY_URL);
737 BIT_SET(e->flag, KEY_CHANGE);
227fbb74 738 }
07622ccd 739 BIT_RESET(e->flag, KEY_PRIVATE);
227fbb74 740 storeHashInsert(e);
741}
742
b8d8561b 743StoreEntry *
4768dbe0 744storeCreateEntry(char *url,
745 char *req_hdr,
746 int req_hdr_sz,
747 int flags,
748 method_t method)
090089c4 749{
090089c4 750 StoreEntry *e = NULL;
30a4f2a8 751 MemObject *mem = NULL;
6602e70e 752 debug(20, 3, "storeCreateEntry: '%s' icp flags=%x\n", url, flags);
090089c4 753
6eb42cae 754 e = new_StoreEntry(WITH_MEMOBJ);
30a4f2a8 755 e->lock_count = 1; /* Note lock here w/o calling storeLock() */
756 mem = e->mem_obj;
090089c4 757 e->url = xstrdup(url);
6eb42cae 758 meta_data.url_strings += strlen(url);
73ddce89 759 e->method = method;
4768dbe0 760 if (req_hdr) {
761 mem->mime_hdr_sz = req_hdr_sz;
762 mem->mime_hdr = xmalloc(req_hdr_sz + 1);
763 xmemcpy(mem->mime_hdr, req_hdr, req_hdr_sz);
764 *(mem->mime_hdr + req_hdr_sz) = '\0';
765 }
234967c9 766 if (BIT_TEST(flags, REQ_CACHABLE)) {
1c481e00 767 BIT_SET(e->flag, ENTRY_CACHABLE);
090089c4 768 BIT_RESET(e->flag, RELEASE_REQUEST);
769 } else {
1c481e00 770 BIT_RESET(e->flag, ENTRY_CACHABLE);
2daae136 771 storeReleaseRequest(e);
090089c4 772 }
234967c9 773 if (BIT_TEST(flags, REQ_HIERARCHICAL))
774 BIT_SET(e->flag, HIERARCHICAL);
775 else
776 BIT_RESET(e->flag, HIERARCHICAL);
777 if (neighbors_do_private_keys || !BIT_TEST(flags, REQ_HIERARCHICAL))
6eb42cae 778 storeSetPrivateKey(e);
090089c4 779 else
6eb42cae 780 storeSetPublicKey(e);
caebbe00 781 BIT_SET(e->flag, ENTRY_HTML);
090089c4 782
234967c9 783 e->store_status = STORE_PENDING;
090089c4 784 storeSetMemStatus(e, NOT_IN_MEMORY);
785 e->swap_status = NO_SWAP;
786 e->swap_file_number = -1;
30a4f2a8 787 mem->data = new_MemObjectData();
090089c4 788 e->refcount = 0;
b8de7ebe 789 e->lastref = squid_curtime;
d1a43e28 790 e->timestamp = 0; /* set in ttlSet() */
30a4f2a8 791 e->ping_status = PING_NONE;
227fbb74 792
090089c4 793 /* allocate pending list */
30a4f2a8 794 mem->pending_list_size = MIN_PENDING;
795 mem->pending = (struct pentry **)
796 xcalloc(mem->pending_list_size, sizeof(struct pentry *));
090089c4 797
798 /* allocate client list */
30a4f2a8 799 mem->client_list_size = MIN_CLIENT;
6e40f263 800 mem->client_list =
30a4f2a8 801 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
2285407f 802 /* storeLog(STORE_LOG_CREATE, e); */
090089c4 803 return e;
804}
805
806/* Add a new object to the cache with empty memory copy and pointer to disk
807 * use to rebuild store from disk. */
24382924 808static StoreEntry *
b8d8561b 809storeAddDiskRestore(char *url, int file_number, int size, time_t expires, time_t timestamp, time_t lastmod)
090089c4 810{
811 StoreEntry *e = NULL;
812
c943f331 813 debug(20, 5, "StoreAddDiskRestore: <URL:%s>: size %d: expires %d: file_number %d\n",
090089c4 814 url, size, expires, file_number);
815
3facc31e 816 /* if you call this you'd better be sure file_number is not
817 * already in use! */
66a91d2d 818
090089c4 819 meta_data.url_strings += strlen(url);
820
227fbb74 821 e = new_StoreEntry(WITHOUT_MEMOBJ);
090089c4 822 e->url = xstrdup(url);
73ddce89 823 e->method = METHOD_GET;
6eb42cae 824 storeSetPublicKey(e);
1c481e00 825 BIT_SET(e->flag, ENTRY_CACHABLE);
090089c4 826 BIT_RESET(e->flag, RELEASE_REQUEST);
6eb42cae 827 BIT_SET(e->flag, ENTRY_HTML);
234967c9 828 e->store_status = STORE_OK;
090089c4 829 storeSetMemStatus(e, NOT_IN_MEMORY);
830 e->swap_status = SWAP_OK;
831 e->swap_file_number = file_number;
832 file_map_bit_set(file_number);
833 e->object_len = size;
834 e->lock_count = 0;
090089c4 835 BIT_RESET(e->flag, CLIENT_ABORT_REQUEST);
836 e->refcount = 0;
3c0117c9 837 e->lastref = timestamp;
f5469bce 838 e->timestamp = timestamp;
839 e->expires = expires;
840 e->lastmod = lastmod;
30a4f2a8 841 e->ping_status = PING_NONE;
090089c4 842 return e;
843}
844
845/* Register interest in an object currently being retrieved. */
b8d8561b 846int
847storeRegister(StoreEntry * e, int fd, PIF handler, void *data)
090089c4 848{
73ddce89 849 PendingEntry *pe = NULL;
850 int old_size;
851 int i;
9dfb6c1c 852 int j;
853 MemObject *mem = e->mem_obj;
090089c4 854
2285407f 855 debug(20, 3, "storeRegister: FD %d '%s'\n", fd, e->key);
090089c4 856
30a4f2a8 857 pe = xcalloc(1, sizeof(PendingEntry));
090089c4 858 pe->fd = fd;
859 pe->handler = handler;
860 pe->data = data;
861
862 /*
863 * I've rewritten all this pendings stuff so that num_pending goes
864 * away, and to fix all of the 'array bounds' problems we were having.
865 * It's now a very simple array, with any NULL slot empty/avail.
866 * If something needs to be added and there are no empty slots,
867 * it'll grow the array.
868 */
869 /* find an empty slot */
9dfb6c1c 870 for (i = 0; i < (int) mem->pending_list_size; i++) {
871 if (mem->pending[i] == NULL)
090089c4 872 break;
9dfb6c1c 873 }
090089c4 874
9dfb6c1c 875 if (i == mem->pending_list_size) {
090089c4 876 /* grow the array */
877 struct pentry **tmp = NULL;
878
9dfb6c1c 879 old_size = mem->pending_list_size;
090089c4 880
881 /* set list_size to an appropriate amount */
9dfb6c1c 882 mem->pending_list_size += MIN_PENDING;
090089c4 883
884 /* allocate, and copy old pending list over to the new one */
9dfb6c1c 885 tmp = xcalloc(mem->pending_list_size, sizeof(struct pentry *));
e62d2dea 886 for (j = 0; j < old_size; j++)
887 tmp[j] = mem->pending[j];
090089c4 888
889 /* free the old list and set the new one */
9dfb6c1c 890 safe_free(mem->pending);
891 mem->pending = tmp;
090089c4 892
9dfb6c1c 893 debug(20, 9, "storeRegister: grew pending list to %d for slot %d.\n",
894 mem->pending_list_size, i);
090089c4 895 }
9dfb6c1c 896 mem->pending[i] = pe;
090089c4 897 return 0;
898}
899
900/* remove handler assoicate to that fd from store pending list */
901/* Also remove entry from client_list if exist. */
902/* return number of successfully free pending entries */
b8d8561b 903int
904storeUnregister(StoreEntry * e, int fd)
090089c4 905{
906 int i;
907 int freed = 0;
6e40f263 908 MemObject *mem = e->mem_obj;
9dfb6c1c 909 debug(20, 9, "storeUnregister: called for FD %d '%s'\n", fd, e->key);
090089c4 910 /* look for entry in client_list */
6e40f263 911 if ((i = storeClientListSearch(mem, fd)) > -1) {
912 safe_free(mem->client_list[i]);
913 mem->client_list[i] = NULL;
090089c4 914 }
915 /* walk the entire list looking for matched fd */
6e40f263 916 for (i = 0; i < (int) mem->pending_list_size; i++) {
917 if (mem->pending[i] == NULL)
918 continue;
919 if (mem->pending[i]->fd != fd)
920 continue;
921 /* found the match fd */
922 safe_free(mem->pending[i]);
923 mem->pending[i] = NULL;
924 freed++;
090089c4 925 }
148a2921 926 if (mem->fd_of_first_client == fd)
927 mem->fd_of_first_client = -1;
9dfb6c1c 928 debug(20, 9, "storeUnregister: returning %d\n", freed);
090089c4 929 return freed;
930}
931
b8d8561b 932int
933storeGetLowestReaderOffset(StoreEntry * entry)
234967c9 934{
30a4f2a8 935 MemObject *mem = entry->mem_obj;
936 int lowest = mem->e_current_len;
234967c9 937 int i;
30a4f2a8 938 for (i = 0; i < mem->client_list_size; i++) {
939 if (mem->client_list[i] == NULL)
234967c9 940 continue;
30a4f2a8 941 if (mem->client_list[i]->last_offset < lowest)
942 lowest = mem->client_list[i]->last_offset;
234967c9 943 }
944 return lowest;
945}
946
090089c4 947/* Call to delete behind upto "target lowest offset"
234967c9 948 * also, update e_lowest_offset */
24382924 949static void
b8d8561b 950storeDeleteBehind(StoreEntry * e)
090089c4 951{
30a4f2a8 952 MemObject *mem = e->mem_obj;
090089c4 953 int free_up_to;
954 int target_offset;
090089c4 955
c943f331 956 debug(20, 3, "storeDeleteBehind: Object: %s\n", e->key);
234967c9 957 debug(20, 3, "storeDeleteBehind: Original Lowest Offset: %d\n",
30a4f2a8 958 mem->e_lowest_offset);
090089c4 959
30a4f2a8 960 free_up_to = mem->e_lowest_offset;
234967c9 961 target_offset = storeGetLowestReaderOffset(e);
090089c4 962
234967c9 963 debug(20, 3, "storeDeleteBehind: target offset: %d\n", target_offset);
090089c4 964 if (target_offset) {
30a4f2a8 965 free_up_to = (int) mem->data->mem_free_data_upto(mem->data, target_offset);
234967c9 966 debug(20, 3, "--> Object is freed upto : %d\n", free_up_to);
30a4f2a8 967 store_mem_size -= free_up_to - mem->e_lowest_offset;
090089c4 968 }
234967c9 969 debug(20, 3, "storeDeleteBehind: New lowest offset: %d\n", free_up_to);
30a4f2a8 970 mem->e_lowest_offset = free_up_to;
090089c4 971}
972
973/* Call handlers waiting for data to be appended to E. */
b8d8561b 974static void
975InvokeHandlers(StoreEntry * e)
090089c4 976{
977 int i;
0701c888 978 int fd;
9dfb6c1c 979 MemObject *mem = e->mem_obj;
63cc9284 980 struct pentry *p = NULL;
981 PIF handler = NULL;
982 void *data = NULL;
090089c4 983 /* walk the entire list looking for valid handlers */
9dfb6c1c 984 for (i = 0; i < (int) mem->pending_list_size; i++) {
63cc9284 985 p = mem->pending[i];
986 if (p == NULL)
987 continue;
988 if ((handler = p->handler) == NULL)
989 continue;
990 data = p->data;
0701c888 991 fd = p->fd;
63cc9284 992 memset(p, '\0', sizeof(struct pentry));
993 safe_free(p);
994 mem->pending[i] = NULL;
0701c888 995 handler(fd, e, data);
090089c4 996 }
090089c4 997}
998
6eb42cae 999/* Mark object as expired */
b8d8561b 1000void
1001storeExpireNow(StoreEntry * e)
9174e204 1002{
2285407f 1003 debug(20, 3, "storeExpireNow: '%s'\n", e->key);
b8de7ebe 1004 e->expires = squid_curtime;
9174e204 1005}
1006
6eb42cae 1007/* switch object to deleting behind mode call by
1008 * retrieval module when object gets too big. */
b8d8561b 1009void
1010storeStartDeleteBehind(StoreEntry * e)
090089c4 1011{
c943f331 1012 debug(20, 2, "storeStartDeleteBehind: Object: %s\n", e->key);
aadcfbad 1013 if (BIT_TEST(e->flag, DELETE_BEHIND))
090089c4 1014 return;
3bbba102 1015 debug(20, 2, "storeStartDeleteBehind:is now in delete behind mode.\n");
1016 /* change its key, so it can't be found by another client */
6eb42cae 1017 storeSetPrivateKey(e);
090089c4 1018 BIT_SET(e->flag, DELETE_BEHIND);
2daae136 1019 storeReleaseRequest(e);
1c481e00 1020 BIT_RESET(e->flag, ENTRY_CACHABLE);
9174e204 1021 storeExpireNow(e);
090089c4 1022}
1023
1024/* Append incoming data from a primary server to an entry. */
b8d8561b 1025void
1026storeAppend(StoreEntry * e, char *data, int len)
090089c4 1027{
85b701ed 1028 /* sanity check */
1029 if (e == NULL) {
1030 debug_trap("storeAppend: NULL entry.");
1031 return;
1032 } else if (e->mem_obj == NULL) {
1033 debug_trap("storeAppend: NULL entry->mem_obj");
1034 return;
1035 } else if (e->mem_obj->data == NULL) {
1036 debug_trap("storeAppend: NULL entry->mem_obj->data");
1037 return;
1038 }
090089c4 1039 if (len) {
2285407f 1040 debug(20, 5, "storeAppend: appending %d bytes for '%s'\n", len, e->key);
d4432957 1041 (void) storeGetMemSpace(len);
090089c4 1042 store_mem_size += len;
e47afcdb 1043 (void) e->mem_obj->data->mem_append(e->mem_obj->data, data, len);
22e4fa85 1044 e->mem_obj->e_current_len += len;
090089c4 1045 }
aadcfbad 1046 if (e->store_status != STORE_ABORTED && !BIT_TEST(e->flag, DELAY_SENDING))
090089c4 1047 InvokeHandlers(e);
090089c4 1048}
1049
24382924 1050#ifdef __STDC__
b8d8561b 1051void
1052storeAppendPrintf(StoreEntry * e, char *fmt,...)
c30c5a73 1053{
15c05bb0 1054 va_list args;
95d659f0 1055 LOCAL_ARRAY(char, buf, 4096);
15c05bb0 1056 va_start(args, fmt);
c30c5a73 1057#else
b8d8561b 1058void
1059storeAppendPrintf(va_alist)
15c05bb0 1060 va_dcl
1061{
1062 va_list args;
1063 StoreEntry *e = NULL;
1064 char *fmt = NULL;
95d659f0 1065 LOCAL_ARRAY(char, buf, 4096);
15c05bb0 1066 va_start(args);
1067 e = va_arg(args, StoreEntry *);
1068 fmt = va_arg(args, char *);
c30c5a73 1069#endif
15c05bb0 1070 buf[0] = '\0';
1071 vsprintf(buf, fmt, args);
1072 storeAppend(e, buf, strlen(buf));
1073 va_end(args);
c30c5a73 1074}
1075
090089c4 1076/* add directory to swap disk */
24382924 1077static int
b8d8561b 1078storeAddSwapDisk(char *path)
090089c4 1079{
a1b0d7cf 1080 char **tmp = NULL;
1081 int i;
1082 if (CacheDirs == NULL) {
1083 CacheDirsAllocated = 4;
1084 CacheDirs = xcalloc(CacheDirsAllocated, sizeof(char *));
1085 }
1086 if (CacheDirsAllocated == ncache_dirs) {
1087 CacheDirsAllocated <<= 1;
1088 tmp = xcalloc(CacheDirsAllocated, sizeof(char *));
fedac7e5 1089 for (i = 0; i < ncache_dirs; i++)
1090 *(tmp + i) = *(CacheDirs + i);
a1b0d7cf 1091 xfree(CacheDirs);
1092 CacheDirs = tmp;
1093 }
fedac7e5 1094 *(CacheDirs + ncache_dirs) = xstrdup(path);
090089c4 1095 return ++ncache_dirs;
1096}
1097
1098/* return the nth swap directory */
b8d8561b 1099char *
1100swappath(int n)
090089c4 1101{
fedac7e5 1102 return *(CacheDirs + (n % ncache_dirs));
090089c4 1103}
1104
1105
1106/* return full name to swapfile */
24382924 1107static char *
b8d8561b 1108storeSwapFullPath(int fn, char *fullpath)
090089c4 1109{
95d659f0 1110 LOCAL_ARRAY(char, fullfilename, MAX_FILE_NAME_LEN);
f5469bce 1111 if (!fullpath)
1112 fullpath = fullfilename;
1113 fullpath[0] = '\0';
1114 sprintf(fullpath, "%s/%02X/%02X/%08X",
090089c4 1115 swappath(fn),
f5469bce 1116 (fn / ncache_dirs) % SWAP_DIRECTORIES_L1,
1117 (fn / ncache_dirs) / SWAP_DIRECTORIES_L1 % SWAP_DIRECTORIES_L2,
090089c4 1118 fn);
f5469bce 1119 return fullpath;
090089c4 1120}
1121
1122/* swapping in handle */
24382924 1123static int
b8d8561b 1124storeSwapInHandle(int fd_notused, char *buf, int len, int flag, StoreEntry * e, int offset_notused)
090089c4 1125{
30a4f2a8 1126 MemObject *mem = e->mem_obj;
1c481e00 1127 SIH handler = NULL;
1128 void *data = NULL;
2285407f 1129 debug(20, 2, "storeSwapInHandle: '%s'\n", e->key);
090089c4 1130
1131 if ((flag < 0) && (flag != DISK_EOF)) {
c943f331 1132 debug(20, 0, "storeSwapInHandle: SwapIn failure (err code = %d).\n", flag);
30a4f2a8 1133 put_free_8k_page(mem->e_swap_buf);
090089c4 1134 storeSetMemStatus(e, NOT_IN_MEMORY);
e47afcdb 1135 file_close(mem->swapin_fd);
090089c4 1136 swapInError(-1, e); /* Invokes storeAbort() and completes the I/O */
d4432957 1137 if ((handler = mem->swapin_complete_handler) != NULL) {
1c481e00 1138 data = mem->swapin_complete_data;
30a4f2a8 1139 mem->swapin_complete_handler = NULL;
1140 mem->swapin_complete_data = NULL;
1c481e00 1141 handler(2, data);
30a4f2a8 1142 }
090089c4 1143 return -1;
1144 }
30a4f2a8 1145 debug(20, 5, "storeSwapInHandle: e->swap_offset = %d\n", mem->swap_offset);
1146 debug(20, 5, "storeSwapInHandle: len = %d\n", len);
1147 debug(20, 5, "storeSwapInHandle: e->e_current_len = %d\n", mem->e_current_len);
1148 debug(20, 5, "storeSwapInHandle: e->object_len = %d\n", e->object_len);
090089c4 1149
1150 /* always call these, even if len == 0 */
30a4f2a8 1151 mem->swap_offset += len;
090089c4 1152 storeAppend(e, buf, len);
1153
30a4f2a8 1154 if (mem->e_current_len < e->object_len && flag != DISK_EOF) {
090089c4 1155 /* some more data to swap in, reschedule */
e47afcdb 1156 file_read(mem->swapin_fd,
30a4f2a8 1157 mem->e_swap_buf,
090089c4 1158 SWAP_BUF,
30a4f2a8 1159 mem->swap_offset,
090089c4 1160 (FILE_READ_HD) storeSwapInHandle,
51496678 1161 (void *) e);
090089c4 1162 } else {
1163 /* complete swapping in */
1164 storeSetMemStatus(e, IN_MEMORY);
30a4f2a8 1165 put_free_8k_page(mem->e_swap_buf);
e47afcdb 1166 file_close(mem->swapin_fd);
2285407f 1167 storeLog(STORE_LOG_SWAPIN, e);
c943f331 1168 debug(20, 5, "storeSwapInHandle: SwapIn complete: <URL:%s> from %s.\n",
090089c4 1169 e->url, storeSwapFullPath(e->swap_file_number, NULL));
30a4f2a8 1170 if (mem->e_current_len != e->object_len) {
c943f331 1171 debug(20, 0, "storeSwapInHandle: WARNING! Object size mismatch.\n");
1172 debug(20, 0, " --> <URL:%s>\n", e->url);
1173 debug(20, 0, " --> Expecting %d bytes from file: %s\n", e->object_len,
090089c4 1174 storeSwapFullPath(e->swap_file_number, NULL));
c943f331 1175 debug(20, 0, " --> Only read %d bytes\n",
30a4f2a8 1176 mem->e_current_len);
1177 }
d4432957 1178 if ((handler = mem->swapin_complete_handler) != NULL) {
1c481e00 1179 data = mem->swapin_complete_data;
30a4f2a8 1180 mem->swapin_complete_handler = NULL;
1181 mem->swapin_complete_data = NULL;
1c481e00 1182 handler(0, data);
090089c4 1183 }
aadcfbad 1184 if (BIT_TEST(e->flag, RELEASE_REQUEST))
090089c4 1185 storeRelease(e);
12f6386d 1186 else {
1187 requestUnlink(mem->request);
1188 mem->request = NULL;
1189 }
090089c4 1190 }
1191 return 0;
1192}
1193
1194/* start swapping in */
b8d8561b 1195static int
1196storeSwapInStart(StoreEntry * e, SIH swapin_complete_handler, void *swapin_complete_data)
090089c4 1197{
1198 int fd;
fc07a0e4 1199 char *path = NULL;
e47afcdb 1200 MemObject *mem = NULL;
090089c4 1201
1202 /* sanity check! */
85b701ed 1203 if (e->swap_status != SWAP_OK) {
1204 debug_trap("storeSwapInStart: bad swap_status");
1205 return -1;
1206 } else if (e->swap_file_number < 0) {
1207 debug_trap("storeSwapInStart: bad swap_file_number");
1208 return -1;
1209 } else if (e->mem_obj) {
1210 debug_trap("storeSwapInStart: mem_obj already present");
1211 return -1;
1212 }
e47afcdb 1213 e->mem_obj = mem = new_MemObject();
090089c4 1214
fc07a0e4 1215 path = storeSwapFullPath(e->swap_file_number, NULL);
0ffd22bc 1216 if ((fd = file_open(path, NULL, O_RDONLY)) < 0) {
30a4f2a8 1217 debug(20, 0, "storeSwapInStart: Failed for '%s'\n", e->url);
090089c4 1218 storeSetMemStatus(e, NOT_IN_MEMORY);
fc07a0e4 1219 /* Invoke a store abort that should free the memory object */
090089c4 1220 return -1;
1221 }
e47afcdb 1222 mem->swapin_fd = (short) fd;
c943f331 1223 debug(20, 5, "storeSwapInStart: initialized swap file '%s' for <URL:%s>\n",
fc07a0e4 1224 path, e->url);
090089c4 1225 storeSetMemStatus(e, SWAPPING_IN);
e47afcdb 1226 mem->data = new_MemObjectData();
1227 mem->swap_offset = 0;
1228 mem->e_swap_buf = get_free_8k_page();
090089c4 1229
1230 /* start swapping daemon */
fc07a0e4 1231 file_read(fd,
e47afcdb 1232 mem->e_swap_buf,
090089c4 1233 SWAP_BUF,
e47afcdb 1234 mem->swap_offset,
090089c4 1235 (FILE_READ_HD) storeSwapInHandle,
51496678 1236 (void *) e);
e47afcdb 1237 mem->swapin_complete_handler = swapin_complete_handler;
1238 mem->swapin_complete_data = swapin_complete_data;
090089c4 1239 return 0;
1240}
1241
b8d8561b 1242static void
1243storeSwapLog(StoreEntry * e)
f5469bce 1244{
0be3ec60 1245 LOCAL_ARRAY(char, logmsg, MAX_URL << 1);
f5469bce 1246 /* Note this printf format appears in storeWriteCleanLog() too */
1247 sprintf(logmsg, "%08x %08x %08x %08x %9d %s\n",
1248 (int) e->swap_file_number,
1249 (int) e->timestamp,
1250 (int) e->expires,
1251 (int) e->lastmod,
1252 e->object_len,
1253 e->url);
1254 file_write(swaplog_fd,
1255 xstrdup(logmsg),
1256 strlen(logmsg),
1257 swaplog_lock,
1258 NULL,
1259 NULL,
1260 xfree);
1261}
1262
24382924 1263static void
b8d8561b 1264storeSwapOutHandle(int fd, int flag, StoreEntry * e)
090089c4 1265{
95d659f0 1266 LOCAL_ARRAY(char, filename, MAX_FILE_NAME_LEN);
e47afcdb 1267 MemObject *mem = e->mem_obj;
090089c4 1268
2285407f 1269 debug(20, 3, "storeSwapOutHandle: '%s'\n", e->key);
3e98df20 1270 if (mem == NULL) {
a1e47288 1271 debug(20, 0, "%s\n", storeToString(e));
85b701ed 1272 debug_trap("Someone is swapping out a bad entry");
3e98df20 1273 return;
1274 }
090089c4 1275 storeSwapFullPath(e->swap_file_number, filename);
090089c4 1276
1277 if (flag < 0) {
c943f331 1278 debug(20, 1, "storeSwapOutHandle: SwapOut failure (err code = %d).\n",
090089c4 1279 flag);
1280 e->swap_status = NO_SWAP;
e47afcdb 1281 put_free_8k_page(mem->e_swap_buf);
090089c4 1282 file_close(fd);
30a4f2a8 1283 storeRelease(e);
090089c4 1284 if (e->swap_file_number != -1) {
1285 file_map_bit_reset(e->swap_file_number);
1286 safeunlink(filename, 0); /* remove it */
1287 e->swap_file_number = -1;
1288 }
1289 if (flag == DISK_NO_SPACE_LEFT) {
1290 /* reduce the swap_size limit to the current size. */
b1c0cc67 1291 Config.Swap.maxSize = store_swap_size;
1292 storeConfigure();
090089c4 1293 }
1294 return;
1295 }
3e98df20 1296 debug(20, 6, "storeSwapOutHandle: e->swap_offset = %d\n", mem->swap_offset);
1297 debug(20, 6, "storeSwapOutHandle: e->e_swap_buf_len = %d\n", mem->e_swap_buf_len);
1298 debug(20, 6, "storeSwapOutHandle: e->object_len = %d\n", e->object_len);
1299 debug(20, 6, "storeSwapOutHandle: store_swap_size = %dk\n", store_swap_size);
090089c4 1300
e47afcdb 1301 mem->swap_offset += mem->e_swap_buf_len;
090089c4 1302 /* round up */
e47afcdb 1303 store_swap_size += ((mem->e_swap_buf_len + 1023) >> 10);
1304 if (mem->swap_offset >= e->object_len) {
090089c4 1305 /* swapping complete */
1306 e->swap_status = SWAP_OK;
e47afcdb 1307 file_close(mem->swapout_fd);
2285407f 1308 storeLog(STORE_LOG_SWAPOUT, e);
c943f331 1309 debug(20, 5, "storeSwapOutHandle: SwapOut complete: <URL:%s> to %s.\n",
090089c4 1310 e->url, storeSwapFullPath(e->swap_file_number, NULL));
e47afcdb 1311 put_free_8k_page(mem->e_swap_buf);
f5469bce 1312 storeSwapLog(e);
a528eb87 1313 HTTPCacheInfo->proto_newobject(HTTPCacheInfo,
e47afcdb 1314 mem->request->protocol,
30a4f2a8 1315 e->object_len,
1316 FALSE);
090089c4 1317 /* check if it's request to be released. */
aadcfbad 1318 if (BIT_TEST(e->flag, RELEASE_REQUEST))
090089c4 1319 storeRelease(e);
56f29785 1320 else if (storeShouldPurgeMem(e))
30a4f2a8 1321 storePurgeMem(e);
12f6386d 1322 else {
1323 requestUnlink(mem->request);
1324 mem->request = NULL;
1325 }
090089c4 1326 return;
1327 }
1328 /* write some more data, reschedule itself. */
3facc31e 1329 storeCopy(e,
e47afcdb 1330 mem->swap_offset,
3facc31e 1331 SWAP_BUF,
e47afcdb 1332 mem->e_swap_buf,
1333 &(mem->e_swap_buf_len));
1334 file_write(mem->swapout_fd,
1335 mem->e_swap_buf,
1336 mem->e_swap_buf_len,
1337 mem->e_swap_access,
3facc31e 1338 storeSwapOutHandle,
e47afcdb 1339 e,
1340 NULL);
090089c4 1341 return;
090089c4 1342}
1343
1344
1345/* start swapping object to disk */
b8d8561b 1346static int
1347storeSwapOutStart(StoreEntry * e)
090089c4 1348{
1349 int fd;
e47afcdb 1350 int x;
6e40f263 1351 LOCAL_ARRAY(char, swapfilename, MAX_FILE_NAME_LEN);
e47afcdb 1352 MemObject *mem = e->mem_obj;
090089c4 1353 /* Suggest a new swap file number */
1354 swapfileno = (swapfileno + 1) % (MAX_SWAP_FILE);
1355 /* Record the number returned */
1356 swapfileno = file_map_allocate(swapfileno);
1357 storeSwapFullPath(swapfileno, swapfilename);
e47afcdb 1358 fd = file_open(swapfilename, NULL, O_WRONLY | O_CREAT | O_TRUNC);
090089c4 1359 if (fd < 0) {
c943f331 1360 debug(20, 0, "storeSwapOutStart: Unable to open swapfile: %s\n",
090089c4 1361 swapfilename);
1362 file_map_bit_reset(swapfileno);
1363 e->swap_file_number = -1;
1364 return -1;
1365 }
e47afcdb 1366 mem->swapout_fd = (short) fd;
c943f331 1367 debug(20, 5, "storeSwapOutStart: Begin SwapOut <URL:%s> to FD %d FILE %s.\n",
090089c4 1368 e->url, fd, swapfilename);
090089c4 1369 e->swap_file_number = swapfileno;
e47afcdb 1370 if ((mem->e_swap_access = file_write_lock(mem->swapout_fd)) < 0) {
c943f331 1371 debug(20, 0, "storeSwapOutStart: Unable to lock swapfile: %s\n",
090089c4 1372 swapfilename);
1373 file_map_bit_reset(e->swap_file_number);
1374 e->swap_file_number = -1;
1375 return -1;
1376 }
1377 e->swap_status = SWAPPING_OUT;
e47afcdb 1378 mem->swap_offset = 0;
1379 mem->e_swap_buf = get_free_8k_page();
1380 mem->e_swap_buf_len = 0;
1381 storeCopy(e,
1382 0,
1383 SWAP_BUF,
1384 mem->e_swap_buf,
a1e47288 1385 &mem->e_swap_buf_len);
090089c4 1386 /* start swapping daemon */
e47afcdb 1387 x = file_write(mem->swapout_fd,
1388 mem->e_swap_buf,
1389 mem->e_swap_buf_len,
1390 mem->e_swap_access,
1391 storeSwapOutHandle,
1392 e,
1393 NULL);
1394 if (x != DISK_OK)
1395 fatal_dump(NULL); /* This shouldn't happen */
090089c4 1396 return 0;
1397}
1398
1399/* recreate meta data from disk image in swap directory */
3facc31e 1400
1401/* Add one swap file at a time from disk storage */
b8d8561b 1402static int
1403storeDoRebuildFromDisk(struct storeRebuild_data *data)
3facc31e 1404{
95d659f0 1405 LOCAL_ARRAY(char, swapfile, MAXPATHLEN);
1406 LOCAL_ARRAY(char, url, MAX_URL + 1);
090089c4 1407 StoreEntry *e = NULL;
090089c4 1408 time_t expires;
1409 time_t timestamp;
f5469bce 1410 time_t lastmod;
1411 int scan1;
1412 int scan2;
1413 int scan3;
1414 int scan4;
3facc31e 1415 struct stat sb;
1416 off_t size;
090089c4 1417 int sfileno = 0;
30a4f2a8 1418 int count;
f5469bce 1419 int x;
30a4f2a8 1420
1421 /* load a number of objects per invocation */
1422 for (count = 0; count < data->speed; count++) {
1423 if (!fgets(data->line_in, 4095, data->log))
e47afcdb 1424 return !diskWriteIsComplete(swaplog_fd); /* We are done */
30a4f2a8 1425
1426 if ((++data->linecount & 0xFFF) == 0)
1427 debug(20, 1, " %7d Lines read so far.\n", data->linecount);
1428
f5469bce 1429 debug(20, 9, "line_in: %s", data->line_in);
30a4f2a8 1430 if ((data->line_in[0] == '\0') || (data->line_in[0] == '\n') ||
1431 (data->line_in[0] == '#'))
1432 continue; /* skip bad lines */
1433
f5469bce 1434 url[0] = '\0';
1435 swapfile[0] = '\0';
1436 sfileno = 0;
1437 scan1 = 0;
1438 scan2 = 0;
30a4f2a8 1439 scan3 = 0;
f5469bce 1440 scan4 = 0;
1441 x = sscanf(data->line_in, "%x %x %x %x %d %s",
0be3ec60 1442 &sfileno, /* swap_file_number */
1443 &scan1, /* timestamp */
1444 &scan2, /* expires */
1445 &scan3, /* last modified */
1446 &scan4, /* size */
1447 url); /* url */
f5469bce 1448 if (x > 0)
0be3ec60 1449 storeSwapFullPath(sfileno, swapfile);
f5469bce 1450 if (x != 6) {
1451 if (opt_unlink_on_reload && swapfile[0])
1452 safeunlink(swapfile, 0);
30a4f2a8 1453 continue;
1454 }
f5469bce 1455 timestamp = (time_t) scan1;
1456 expires = (time_t) scan2;
1457 lastmod = (time_t) scan3;
1458 size = (off_t) scan4;
30a4f2a8 1459
1460 if (store_rebuilding != STORE_REBUILDING_FAST) {
1461 if (stat(swapfile, &sb) < 0) {
79b5cc5f 1462 if (squid_curtime - expires > Config.expireAge) {
30a4f2a8 1463 debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
1464 if (opt_unlink_on_reload)
1465 safeunlink(swapfile, 1);
1466 data->expcount++;
1467 } else {
1468 debug(20, 3, "storeRebuildFromDisk: Swap file missing: <URL:%s>: %s: %s.\n", url, swapfile, xstrerror());
1469 if (opt_unlink_on_reload)
f5469bce 1470 safeunlink(swapfile, 1);
30a4f2a8 1471 }
1472 continue;
1473 }
1474 /* Empty swap file? */
1475 if (sb.st_size == 0) {
30731487 1476 if (opt_unlink_on_reload)
f5469bce 1477 safeunlink(swapfile, 1);
30a4f2a8 1478 continue;
090089c4 1479 }
d1a43e28 1480#ifdef DONT_DO_THIS
30a4f2a8 1481 /* timestamp might be a little bigger than sb.st_mtime */
1482 delta = (int) (timestamp - sb.st_mtime);
1483 if (delta > REBUILD_TIMESTAMP_DELTA_MAX || delta < 0) {
1484 /* this log entry doesn't correspond to this file */
1485 data->clashcount++;
1486 continue;
1487 }
d1a43e28 1488#endif
30a4f2a8 1489 /* Wrong size? */
1490 if (sb.st_size != size) {
1491 /* this log entry doesn't correspond to this file */
1492 data->clashcount++;
1493 continue;
1494 }
d1a43e28 1495#ifdef DONT_DO_THIS
30a4f2a8 1496 timestamp = sb.st_mtime;
d1a43e28 1497#endif
9dfb6c1c 1498 debug(20, 9, "storeRebuildFromDisk: swap file exists: <URL:%s>: %s\n",
30a4f2a8 1499 url, swapfile);
3facc31e 1500 }
30a4f2a8 1501 if ((e = storeGet(url))) {
1502 if (e->timestamp > timestamp) {
1503 /* already have a newer object in memory, throw old one away */
1504 debug(20, 3, "storeRebuildFromDisk: Replaced: %s\n", url);
1505 if (opt_unlink_on_reload)
1506 safeunlink(swapfile, 1);
1507 data->dupcount++;
1508 continue;
1509 }
1510 debug(20, 6, "storeRebuildFromDisk: Duplicate: <URL:%s>\n", url);
1511 storeRelease(e);
1512 data->objcount--;
1513 data->dupcount++;
090089c4 1514 }
79b5cc5f 1515 if (squid_curtime - expires > Config.expireAge) {
30a4f2a8 1516 debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
30731487 1517 if (opt_unlink_on_reload)
d68adf86 1518 safeunlink(swapfile, 1);
30a4f2a8 1519 data->expcount++;
1520 continue;
090089c4 1521 }
30a4f2a8 1522 /* Is the swap file number already taken? */
1523 if (file_map_bit_test(sfileno)) {
1524 /* Yes it is, we can't use this swapfile */
1525 debug(20, 2, "storeRebuildFromDisk: Line %d Active clash: file #%d\n",
1526 data->linecount,
1527 sfileno);
1528 debug(20, 3, "storeRebuildFromDisk: --> <URL:%s>\n", url);
1529 /* don't unlink the file! just skip this log entry */
1530 data->clashcount++;
1531 continue;
1532 }
1533 /* update store_swap_size */
1534 store_swap_size += (int) ((size + 1023) >> 10);
1535 data->objcount++;
f5469bce 1536 e = storeAddDiskRestore(url,
30a4f2a8 1537 sfileno,
1538 (int) size,
1539 expires,
f5469bce 1540 timestamp,
1541 lastmod);
1542 storeSwapLog(e);
a528eb87 1543 HTTPCacheInfo->proto_newobject(HTTPCacheInfo,
1ac9bc38 1544 urlParseProtocol(url),
30a4f2a8 1545 (int) size,
1546 TRUE);
c943f331 1547 }
3facc31e 1548 return 1;
1549}
1550
1551/* meta data recreated from disk image in swap directory */
b8d8561b 1552static void
1553storeRebuiltFromDisk(struct storeRebuild_data *data)
3facc31e 1554{
1555 time_t r;
1556 time_t stop;
c943f331 1557
b37a37f9 1558 stop = getCurrentTime();
3facc31e 1559 r = stop - data->start;
c943f331 1560 debug(20, 1, "Finished rebuilding storage from disk image.\n");
3facc31e 1561 debug(20, 1, " %7d Lines read from previous logfile.\n", data->linecount);
1562 debug(20, 1, " %7d Objects loaded.\n", data->objcount);
1563 debug(20, 1, " %7d Objects expired.\n", data->expcount);
1564 debug(20, 1, " %7d Duplicate URLs purged.\n", data->dupcount);
1565 debug(20, 1, " %7d Swapfile clashes avoided.\n", data->clashcount);
c943f331 1566 debug(20, 1, " Took %d seconds (%6.1lf objects/sec).\n",
3facc31e 1567 r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1));
c943f331 1568 debug(20, 1, " store_swap_size = %dk\n", store_swap_size);
090089c4 1569
30a4f2a8 1570 store_rebuilding = STORE_NOT_REBUILDING;
3facc31e 1571
1572 fclose(data->log);
1573 safe_free(data);
1574 sprintf(tmp_filename, "%s.new", swaplog_file);
1575 if (rename(tmp_filename, swaplog_file) < 0) {
9a70f206 1576 debug(20, 0, "storeRebuiltFromDisk: %s,%s: %s\n",
9ff15015 1577 tmp_filename, swaplog_file, xstrerror());
9a70f206 1578 fatal_dump("storeRebuiltFromDisk: rename failed");
3facc31e 1579 }
9a70f206 1580 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK)
1581 fatal_dump("storeRebuiltFromDisk: swaplog unlock failed");
2c0a39aa 1582 file_close(swaplog_fd);
e47afcdb 1583 if ((swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT)) < 0)
9a70f206 1584 fatal_dump("storeRebuiltFromDisk: file_open(swaplog_file) failed");
e83421f8 1585 swaplog_lock = file_write_lock(swaplog_fd);
090089c4 1586}
1587
24382924 1588static void
b8d8561b 1589storeStartRebuildFromDisk(void)
3facc31e 1590{
1591 struct stat sb;
1592 int i;
1593 struct storeRebuild_data *data;
1594 time_t last_clean;
1595
1596 if (stat(swaplog_file, &sb) < 0) {
1597 debug(20, 1, "storeRebuildFromDisk: No log file\n");
30a4f2a8 1598 store_rebuilding = STORE_NOT_REBUILDING;
3facc31e 1599 return;
1600 }
1601 data = xcalloc(1, sizeof(*data));
1602
6e40f263 1603 for (i = 0; i < ncache_dirs; i++)
3facc31e 1604 debug(20, 1, "Rebuilding storage from disk image in %s\n", swappath(i));
1605 data->start = getCurrentTime();
1606
1607 /* Check if log is clean */
1608 sprintf(tmp_filename, "%s/log-last-clean", swappath(0));
1609 if (stat(tmp_filename, &sb) >= 0) {
1610 last_clean = sb.st_mtime;
9a70f206 1611 if (stat(swaplog_file, &sb) >= 0)
30a4f2a8 1612 store_rebuilding = (sb.st_mtime <= last_clean) ?
1613 STORE_REBUILDING_FAST : STORE_REBUILDING_SLOW;
3facc31e 1614 }
30a4f2a8 1615 /* Remove timestamp in case we crash during rebuild */
1616 safeunlink(tmp_filename, 1);
3facc31e 1617 /* close the existing write-only swaplog, and open a temporary
1618 * write-only swaplog */
1619 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK)
1620 fatal_dump("storeStartRebuildFromDisk: swaplog unlock failed");
1621 if (swaplog_fd > -1)
1622 file_close(swaplog_fd);
1623 sprintf(tmp_filename, "%s.new", swaplog_file);
e47afcdb 1624 swaplog_fd = file_open(tmp_filename, NULL, O_WRONLY | O_CREAT | O_TRUNC);
66a91d2d 1625 debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, tmp_filename);
3facc31e 1626 if (swaplog_fd < 0) {
1627 debug(20, 0, "storeStartRebuildFromDisk: %s: %s\n",
1628 tmp_filename, xstrerror());
1629 fatal("storeStartRebuildFromDisk: Can't open tmp swaplog");
1630 }
1631 swaplog_lock = file_write_lock(swaplog_fd);
3facc31e 1632 /* Open the existing swap log for reading */
1633 if ((data->log = fopen(swaplog_file, "r")) == (FILE *) NULL) {
1634 sprintf(tmp_error_buf, "storeRebuildFromDisk: %s: %s",
1635 swaplog_file, xstrerror());
1636 fatal(tmp_error_buf);
1637 }
66a91d2d 1638 debug(20, 3, "data->log %d is now '%s'\n", fileno(data->log), swaplog_file);
30a4f2a8 1639 if (store_rebuilding == STORE_REBUILDING_FAST)
3facc31e 1640 debug(20, 1, "Rebuilding in FAST MODE.\n");
1641
1642 memset(data->line_in, '\0', 4096);
30a4f2a8 1643 data->speed = store_rebuilding == STORE_REBUILDING_FAST ? 50 : 5;
3facc31e 1644
1645 /* Start reading the log file */
30a4f2a8 1646 if (opt_foreground_rebuild) {
1647 while (storeDoRebuildFromDisk(data));
1648 storeRebuiltFromDisk(data);
1649 } else {
1650 runInBackground("storeRebuild",
1651 (int (*)(void *)) storeDoRebuildFromDisk,
1652 data,
1653 (void (*)(void *)) storeRebuiltFromDisk);
1654 }
3facc31e 1655}
090089c4 1656
1657/* return current swap size in kilo-bytes */
b8d8561b 1658int
1659storeGetSwapSize(void)
090089c4 1660{
1661 return store_swap_size;
1662}
1663
b8d8561b 1664static int
1665storeCheckSwapable(StoreEntry * e)
6602e70e 1666{
79b5cc5f 1667 if (squid_curtime - e->expires > Config.expireAge) {
3e98df20 1668 debug(20, 2, "storeCheckSwapable: NO: expires now\n");
73ddce89 1669 } else if (e->method != METHOD_GET) {
2285407f 1670 debug(20, 2, "storeCheckSwapable: NO: non-GET method\n");
1c481e00 1671 } else if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) {
2285407f 1672 debug(20, 2, "storeCheckSwapable: NO: not cachable\n");
6602e70e 1673 } else if (BIT_TEST(e->flag, RELEASE_REQUEST)) {
2285407f 1674 debug(20, 2, "storeCheckSwapable: NO: release requested\n");
6602e70e 1675 } else if (!storeEntryValidLength(e)) {
2285407f 1676 debug(20, 2, "storeCheckSwapable: NO: wrong content-length\n");
79b5cc5f 1677 } else if (BIT_TEST(e->flag, ENTRY_NEGCACHED)) {
1678 debug(20, 2, "storeCheckSwapable: NO: negative cached\n");
3e98df20 1679 return 0; /* avoid release call below */
d4432957 1680 } else {
6602e70e 1681 return 1;
d4432957 1682 }
6602e70e 1683
2daae136 1684 storeReleaseRequest(e);
1c481e00 1685 BIT_RESET(e->flag, ENTRY_CACHABLE);
6602e70e 1686 return 0;
1687}
1688
1689
090089c4 1690
1691/* Complete transfer into the local cache. */
b8d8561b 1692void
1693storeComplete(StoreEntry * e)
090089c4 1694{
2285407f 1695 debug(20, 3, "storeComplete: '%s'\n", e->key);
22e4fa85 1696 e->object_len = e->mem_obj->e_current_len;
090089c4 1697 InvokeHandlers(e);
b8de7ebe 1698 e->lastref = squid_curtime;
234967c9 1699 e->store_status = STORE_OK;
090089c4 1700 storeSetMemStatus(e, IN_MEMORY);
1701 e->swap_status = NO_SWAP;
22e4fa85 1702 safe_free(e->mem_obj->mime_hdr);
aadcfbad 1703 if (BIT_TEST(e->flag, RELEASE_REQUEST))
090089c4 1704 storeRelease(e);
12f6386d 1705 else if (storeCheckSwapable(e))
1706 storeSwapOutStart(e);
1707 else {
1708 requestUnlink(e->mem_obj->request);
1709 e->mem_obj->request = NULL;
1710 }
090089c4 1711}
1712
1713/*
1714 * Fetch aborted. Tell all clients to go home. Negatively cache
1715 * abort message, freeing the data for this object
1716 */
b8d8561b 1717void
1718storeAbort(StoreEntry * e, char *msg)
090089c4 1719{
95d659f0 1720 LOCAL_ARRAY(char, mime_hdr, 300);
caebbe00 1721 char *abort_msg;
3e98df20 1722 MemObject *mem = e->mem_obj;
090089c4 1723
85b701ed 1724 if (e->store_status != STORE_PENDING) { /* XXX remove later */
1725 debug_trap("storeAbort: bad store_status");
1726 return;
fedac7e5 1727 } else if (mem == NULL) { /* XXX remove later */
85b701ed 1728 debug_trap("storeAbort: null mem");
1729 return;
1730 }
2285407f 1731 debug(20, 6, "storeAbort: '%s'\n", e->key);
79b5cc5f 1732 storeNegativeCache(e);
234967c9 1733 e->store_status = STORE_ABORTED;
090089c4 1734 storeSetMemStatus(e, IN_MEMORY);
1735 /* No DISK swap for negative cached object */
1736 e->swap_status = NO_SWAP;
b8de7ebe 1737 e->lastref = squid_curtime;
090089c4 1738 /* In case some parent responds late and
1739 * tries to restart the fetch, say that it's been
1740 * dispatched already.
1741 */
6eb42cae 1742 BIT_SET(e->flag, ENTRY_DISPATCHED);
090089c4 1743
30a4f2a8 1744 storeLockObject(e, NULL, NULL);
090089c4 1745
1746 /* Count bytes faulted through cache but not moved to disk */
a528eb87 1747 HTTPCacheInfo->proto_touchobject(HTTPCacheInfo,
3e98df20 1748 mem->request ? mem->request->protocol : PROTO_NONE,
1749 mem->e_current_len);
090089c4 1750 if (msg) {
caebbe00 1751 abort_msg = get_free_8k_page();
1752 strcpy(abort_msg, "HTTP/1.0 400 Cache Detected Error\r\n");
1753 mk_mime_hdr(mime_hdr,
1754 "text/html",
1755 strlen(msg),
1756 (time_t) Config.negativeTtl,
1757 squid_curtime);
1758 strcat(abort_msg, mime_hdr);
aadcfbad 1759 strcat(abort_msg, "\r\n");
caebbe00 1760 strncat(abort_msg, msg, 8191 - strlen(abort_msg));
090089c4 1761 storeAppend(e, abort_msg, strlen(abort_msg));
a1e47288 1762 safe_free(mem->e_abort_msg);
3e98df20 1763 mem->e_abort_msg = xstrdup(abort_msg);
090089c4 1764 /* Set up object for negative caching */
1765 BIT_SET(e->flag, ABORT_MSG_PENDING);
caebbe00 1766 put_free_8k_page(abort_msg);
090089c4 1767 }
1768 /* We assign an object length here--The only other place we assign the
1769 * object length is in storeComplete() */
3e98df20 1770 e->object_len = mem->e_current_len;
090089c4 1771
1772 /* Call handlers so they can report error. */
1773 InvokeHandlers(e);
1774
1775 storeUnlockObject(e);
85b701ed 1776 return;
090089c4 1777}
1778
090089c4 1779/* get the first in memory object entry in the storage */
24382924 1780static StoreEntry *
b8d8561b 1781storeGetInMemFirst(void)
090089c4 1782{
1783 hash_link *first = NULL;
c3c5b744 1784 first = hash_first(in_mem_table);
090089c4 1785 return (first ? ((StoreEntry *) first->item) : NULL);
1786}
1787
1788
1789/* get the next in memory object entry in the storage for a given
1790 * search pointer */
24382924 1791static StoreEntry *
b8d8561b 1792storeGetInMemNext(void)
090089c4 1793{
1794 hash_link *next = NULL;
c3c5b744 1795 next = hash_next(in_mem_table);
090089c4 1796 return (next ? ((StoreEntry *) next->item) : NULL);
1797}
1798
1799/* get the first entry in the storage */
b8d8561b 1800StoreEntry *
1801storeGetFirst(void)
090089c4 1802{
c3c5b744 1803 return ((StoreEntry *) hash_first(store_table));
090089c4 1804}
1805
1806
1807/* get the next entry in the storage for a given search pointer */
b8d8561b 1808StoreEntry *
1809storeGetNext(void)
090089c4 1810{
c3c5b744 1811 return ((StoreEntry *) hash_next(store_table));
090089c4 1812}
1813
620da955 1814/* free up all ttl-expired objects */
b8d8561b 1815int
1816storePurgeOld(void)
090089c4 1817{
1818 StoreEntry *e = NULL;
090089c4 1819 int n = 0;
620da955 1820 int count = 0;
090089c4 1821 for (e = storeGetFirst(); e; e = storeGetNext()) {
0a5b9b32 1822 if ((++n & 0xFF) == 0) {
b37a37f9 1823 getCurrentTime();
0a5b9b32 1824 if (shutdown_pending || reread_pending)
1825 break;
1826 }
090089c4 1827 if ((n & 0xFFF) == 0)
c943f331 1828 debug(20, 2, "storeWalkThrough: %7d objects so far.\n", n);
620da955 1829 if (storeCheckExpired(e))
1830 if (storeRelease(e) == 0)
1831 count++;
090089c4 1832 }
1833 return count;
1834}
1835
1836
090089c4 1837/* Clear Memory storage to accommodate the given object len */
d4432957 1838static int
1839storeGetMemSpace(int size)
090089c4 1840{
b32508fb 1841 StoreEntry *e = NULL;
1842 StoreEntry **list = NULL;
1843 int list_count = 0;
090089c4 1844 int n_expired = 0;
66c2848f 1845 int n_purged = 0;
579a45cd 1846 int n_released = 0;
3bbba102 1847 int n_locked = 0;
090089c4 1848 int i;
b32508fb 1849 static time_t last_warning = 0;
1850 static time_t last_check = 0;
e954773d 1851 int pages_needed;
090089c4 1852
b32508fb 1853 if (squid_curtime == last_check)
090089c4 1854 return 0;
b32508fb 1855 last_check = squid_curtime;
e954773d 1856 pages_needed = (size / SM_PAGE_SIZE) + 1;
1857 if (sm_stats.n_pages_in_use + pages_needed < sm_stats.max_pages)
56f29785 1858 return 0;
1859 if (store_rebuilding == STORE_REBUILDING_FAST)
b32508fb 1860 return 0;
e954773d 1861 debug(20, 2, "storeGetMemSpace: Starting, need %d pages\n", pages_needed);
090089c4 1862
579a45cd 1863 list = xcalloc(meta_data.mem_obj_count, sizeof(ipcache_entry *));
090089c4 1864 for (e = storeGetInMemFirst(); e; e = storeGetInMemNext()) {
579a45cd 1865 if (list_count == meta_data.mem_obj_count)
b32508fb 1866 break;
620da955 1867 if (storeCheckExpired(e)) {
c943f331 1868 debug(20, 2, "storeGetMemSpace: Expired: %s\n", e->url);
090089c4 1869 n_expired++;
090089c4 1870 storeRelease(e);
579a45cd 1871 } else if (storeCheckPurgeMem(e)) {
1872 *(list + list_count) = e;
1873 list_count++;
5e79098a 1874 } else if (!storeEntryLocked(e)) {
579a45cd 1875 *(list + list_count) = e;
1876 list_count++;
1877 } else {
1878 n_locked++;
090089c4 1879 continue;
1880 }
b32508fb 1881 debug(20, 3, "storeGetMemSpace: Adding '%s'\n", e->url);
090089c4 1882 }
b32508fb 1883 debug(20, 5, "storeGetMemSpace: Sorting LRU_list: %7d items\n", list_count);
1884 qsort((char *) list,
1885 list_count,
6e40f263 1886 sizeof(StoreEntry *),
b32508fb 1887 (QS) compareSize);
090089c4 1888
1889 /* Kick LRU out until we have enough memory space */
b32508fb 1890 for (i = 0; i < list_count; i++) {
e954773d 1891 if (sm_stats.n_pages_in_use + pages_needed < store_pages_low)
9d90e665 1892 break;
579a45cd 1893 e = *(list + i);
1894 if (storeCheckPurgeMem(e)) {
5e79098a 1895 storePurgeMem(e);
1896 n_purged++;
579a45cd 1897 } else if (!storeEntryLocked(e)) {
e954773d 1898 /* These will be neg-cached objects */
5e79098a 1899 storeRelease(e);
1900 n_released++;
579a45cd 1901 } else {
85b701ed 1902 debug_trap("storeGetMemSpace: Bad Entry in LRU list");
579a45cd 1903 }
090089c4 1904 }
1905
579a45cd 1906 i = 3;
e954773d 1907 if (sm_stats.n_pages_in_use + pages_needed > store_pages_high) {
3bbba102 1908 if (squid_curtime - last_warning > 600) {
e954773d 1909 debug(20, 0, "WARNING: Over store_pages high-water mark (%d > %d)\n",
1910 sm_stats.n_pages_in_use + pages_needed, store_pages_high);
b32508fb 1911 last_warning = squid_curtime;
e954773d 1912 debug(20, 0, "Perhaps you should increase cache_mem?\n");
579a45cd 1913 i = 0;
090089c4 1914 }
090089c4 1915 }
579a45cd 1916 debug(20, i, "storeGetMemSpace stats:\n");
1917 debug(20, i, " %6d objects locked in memory\n", n_locked);
1918 debug(20, i, " %6d LRU candidates\n", list_count);
1919 debug(20, i, " %6d were purged\n", n_purged);
1920 debug(20, i, " %6d were released\n", n_released);
3d700ed6 1921 xfree(list);
090089c4 1922 return 0;
1923}
1924
b8d8561b 1925static int
1926compareSize(StoreEntry ** e1, StoreEntry ** e2)
090089c4 1927{
6e40f263 1928 if (!e1 || !e2)
1929 fatal_dump(NULL);
22e4fa85 1930 if ((*e1)->mem_obj->e_current_len > (*e2)->mem_obj->e_current_len)
090089c4 1931 return (1);
22e4fa85 1932 if ((*e1)->mem_obj->e_current_len < (*e2)->mem_obj->e_current_len)
090089c4 1933 return (-1);
090089c4 1934 return (0);
1935}
1936
b8d8561b 1937static int
1938compareLastRef(StoreEntry ** e1, StoreEntry ** e2)
090089c4 1939{
1940 if (!e1 || !e2)
1941 fatal_dump(NULL);
090089c4 1942 if ((*e1)->lastref > (*e2)->lastref)
1943 return (1);
090089c4 1944 if ((*e1)->lastref < (*e2)->lastref)
1945 return (-1);
090089c4 1946 return (0);
1947}
1948
1949/* returns the bucket number to work on,
1950 * pointer to next bucket after each calling
1951 */
24382924 1952static unsigned int
b8d8561b 1953storeGetBucketNum(void)
090089c4 1954{
1955 static unsigned int bucket = 0;
090089c4 1956 if (bucket >= STORE_BUCKETS)
1957 bucket = 0;
1958 return (bucket++);
1959}
1960
1961#define SWAP_LRUSCAN_BLOCK 16
1962#define SWAP_MAX_HELP STORE_BUCKETS/2
1963
1964/* The maximum objects to scan for maintain storage space */
a1b0d7cf 1965#define SWAP_LRUSCAN_COUNT 256
1966#define SWAP_LRU_REMOVE_COUNT 8
090089c4 1967
1968/* Clear Swap storage to accommodate the given object len */
b8d8561b 1969int
1970storeGetSwapSpace(int size)
090089c4 1971{
1972 static int fReduceSwap = 0;
1973 static int swap_help = 0;
a1b0d7cf 1974 StoreEntry *e = NULL;
090089c4 1975 int scanned = 0;
1976 int removed = 0;
1977 int expired = 0;
1978 int locked = 0;
1979 int locked_size = 0;
a1b0d7cf 1980 int list_count = 0;
1981 int scan_count = 0;
1982 int max_list_count = SWAP_LRUSCAN_COUNT << 1;
090089c4 1983 int i;
a1b0d7cf 1984 StoreEntry **LRU_list;
090089c4 1985 hash_link *link_ptr = NULL, *next = NULL;
1986 unsigned int kb_size = ((size + 1023) >> 10);
1987
1988 if (store_swap_size + kb_size <= store_swap_low)
1989 fReduceSwap = 0;
1990 if (!fReduceSwap && (store_swap_size + kb_size <= store_swap_high)) {
1991 return 0;
1992 }
c943f331 1993 debug(20, 2, "storeGetSwapSpace: Starting...\n");
090089c4 1994
1995 /* Set flag if swap size over high-water-mark */
1996 if (store_swap_size + kb_size > store_swap_high)
1997 fReduceSwap = 1;
1998
c943f331 1999 debug(20, 2, "storeGetSwapSpace: Need %d bytes...\n", size);
090089c4 2000
a1b0d7cf 2001 LRU_list = xcalloc(max_list_count, sizeof(StoreEntry *));
2002 /* remove expired objects until recover enough or no expired objects */
6e40f263 2003 for (i = 0; i < STORE_BUCKETS; i++) {
090089c4 2004 int expired_in_one_bucket = 0;
ec74cf5f 2005 link_ptr = hash_get_bucket(store_table, storeGetBucketNum());
090089c4 2006 if (link_ptr == NULL)
2007 continue;
2008 /* this while loop handles one bucket of hash table */
2009 expired_in_one_bucket = 0;
79b5cc5f 2010 for (; link_ptr; link_ptr = next) {
a1b0d7cf 2011 if (list_count == max_list_count)
2012 break;
090089c4 2013 scanned++;
2014 next = link_ptr->next;
2015 e = (StoreEntry *) link_ptr;
79b5cc5f 2016 if (storeCheckExpired(e)) {
a1b0d7cf 2017 debug(20, 3, "storeGetSwapSpace: Expired '%s'\n", e->url);
ff5731b1 2018 ++expired_in_one_bucket;
2019 storeRelease(e);
79b5cc5f 2020 } else if (!storeEntryLocked(e)) {
fedac7e5 2021 *(LRU_list + list_count) = e;
a1b0d7cf 2022 list_count++;
2023 scan_count++;
090089c4 2024 } else {
ff5731b1 2025 locked++;
2026 locked_size += e->mem_obj->e_current_len;
090089c4 2027 }
090089c4 2028 } /* while, end of one bucket of hash table */
090089c4 2029 expired += expired_in_one_bucket;
2030 if (expired_in_one_bucket &&
2031 ((!fReduceSwap && (store_swap_size + kb_size <= store_swap_high)) ||
2032 (fReduceSwap && (store_swap_size + kb_size <= store_swap_low)))
2033 ) {
2034 fReduceSwap = 0;
a1b0d7cf 2035 safe_free(LRU_list);
c943f331 2036 debug(20, 2, "storeGetSwapSpace: Finished, %d objects expired.\n",
090089c4 2037 expired);
2038 return 0;
2039 }
a1b0d7cf 2040 qsort((char *) LRU_list,
2041 list_count,
6e40f263 2042 sizeof(StoreEntry *),
2043 (QS) compareLastRef);
85b701ed 2044 if (list_count > SWAP_LRU_REMOVE_COUNT)
fedac7e5 2045 list_count = SWAP_LRU_REMOVE_COUNT; /* chop list */
a1b0d7cf 2046 if (scan_count > SWAP_LRUSCAN_COUNT)
090089c4 2047 break;
2048 } /* for */
2049
30a4f2a8 2050#ifdef LOTSA_DEBUGGING
090089c4 2051 /* end of candidate selection */
30a4f2a8 2052 debug(20, 2, "storeGetSwapSpace: Current Size: %7d kbytes\n",
2053 store_swap_size);
2054 debug(20, 2, "storeGetSwapSpace: High W Mark: %7d kbytes\n",
2055 store_swap_high);
2056 debug(20, 2, "storeGetSwapSpace: Low W Mark: %7d kbytes\n",
2057 store_swap_low);
2058 debug(20, 2, "storeGetSwapSpace: Entry count: %7d items\n",
2059 meta_data.store_entries);
2060 debug(20, 2, "storeGetSwapSpace: Visited: %7d buckets\n",
2061 i + 1);
2062 debug(20, 2, "storeGetSwapSpace: Scanned: %7d items\n",
2063 scanned);
2064 debug(20, 2, "storeGetSwapSpace: Expired: %7d items\n",
2065 expired);
2066 debug(20, 2, "storeGetSwapSpace: Locked: %7d items\n",
2067 locked);
2068 debug(20, 2, "storeGetSwapSpace: Locked Space: %7d bytes\n",
2069 locked_size);
2070 debug(20, 2, "storeGetSwapSpace: Scan in array: %7d bytes\n",
2071 scan_in_objs);
2072 debug(20, 2, "storeGetSwapSpace: LRU candidate: %7d items\n",
2073 LRU_list->index);
2074#endif /* LOTSA_DEBUGGING */
090089c4 2075
a1b0d7cf 2076 /* Kick LRU out until we have enough swap space */
2077 for (i = 0; i < list_count; i++) {
fedac7e5 2078 if (storeRelease(*(LRU_list + i)) == 0)
a1b0d7cf 2079 removed++;
090089c4 2080 }
a1b0d7cf 2081 if (store_swap_size + kb_size <= store_swap_low)
fedac7e5 2082 fReduceSwap = 0;
a1b0d7cf 2083 debug(20, 2, "storeGetSwapSpace: After Freeing Size: %7d kbytes\n",
2084 store_swap_size);
090089c4 2085 /* free the list */
a1b0d7cf 2086 safe_free(LRU_list);
090089c4 2087
2088 if ((store_swap_size + kb_size > store_swap_high)) {
2089 if (++swap_help > SWAP_MAX_HELP) {
c943f331 2090 debug(20, 0, "storeGetSwapSpace: Nothing to free with %d Kbytes in use.\n",
090089c4 2091 store_swap_size);
c943f331 2092 debug(20, 0, "--> Asking for %d bytes\n", size);
2093 debug(20, 0, "WARNING! Repeated failures to allocate swap space!\n");
2094 debug(20, 0, "WARNING! Please check your disk space.\n");
090089c4 2095 swap_help = 0;
2096 } else {
c943f331 2097 debug(20, 2, "storeGetSwapSpace: Nothing to free with %d Kbytes in use.\n",
090089c4 2098 store_swap_size);
c943f331 2099 debug(20, 2, "--> Asking for %d bytes\n", size);
090089c4 2100 }
2101 } else {
2102 swap_help = 0;
2103 }
2104
30a4f2a8 2105 getCurrentTime(); /* we may have taken more than one second */
2106 debug(20, 2, "Removed %d objects\n", removed);
090089c4 2107 return 0;
2108}
2109
2110
2111/* release an object from a cache */
2112/* return 0 when success. */
b8d8561b 2113int
2114storeRelease(StoreEntry * e)
090089c4 2115{
2116 StoreEntry *result = NULL;
2117 StoreEntry *head_result = NULL;
2118 hash_link *hptr = NULL;
227fbb74 2119 hash_link *head_table_entry = NULL;
090089c4 2120
6602e70e 2121 debug(20, 3, "storeRelease: Releasing: '%s'\n", e->key);
2122
090089c4 2123 /* If, for any reason we can't discard this object because of an
2124 * outstanding request, mark it for pending release */
2125 if (storeEntryLocked(e)) {
9174e204 2126 storeExpireNow(e);
6602e70e 2127 debug(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit\n");
2daae136 2128 storeReleaseRequest(e);
090089c4 2129 return -1;
2130 }
227fbb74 2131 if (e->key != NULL) {
ec74cf5f 2132 if ((hptr = hash_lookup(store_table, e->key)) == NULL) {
2285407f 2133 debug(20, 0, "storeRelease: Not Found: '%s'\n", e->key);
227fbb74 2134 debug(20, 0, "Dump of Entry 'e':\n %s\n", storeToString(e));
0673c0ba 2135 debug_trap("storeRelease: Invalid Entry");
caebbe00 2136 return -1;
227fbb74 2137 }
2138 result = (StoreEntry *) hptr;
2139 if (result != e) {
2140 debug(20, 0, "storeRelease: Duplicated entry? <URL:%s>\n",
2141 result->url ? result->url : "NULL");
2142 debug(20, 0, "Dump of Entry 'e':\n%s", storeToString(e));
2143 debug(20, 0, "Dump of Entry 'result':\n%s", storeToString(result));
0673c0ba 2144 debug_trap("storeRelease: Duplicate Entry");
caebbe00 2145 return -1;
227fbb74 2146 }
090089c4 2147 }
73ddce89 2148 if (e->method == METHOD_GET) {
090089c4 2149 /* check if coresponding HEAD object exists. */
ec74cf5f 2150 head_table_entry = hash_lookup(store_table,
6eb42cae 2151 storeGeneratePublicKey(e->url, METHOD_HEAD));
090089c4 2152 if (head_table_entry) {
2153 head_result = (StoreEntry *) head_table_entry;
2154 if (head_result) {
2155 /* recursive call here to free up /head/ */
2156 storeRelease(head_result);
2157 }
2158 }
2159 }
30a4f2a8 2160 if (store_rebuilding == STORE_REBUILDING_FAST) {
2161 debug(20, 2, "storeRelease: Delaying release until store is rebuilt: '%s'\n",
2162 e->key ? e->key : e->url ? e->url : "NO URL");
2163 storeExpireNow(e);
2164 storeSetPrivateKey(e);
2165 return -1;
2166 }
227fbb74 2167 if (e->key)
2168 debug(20, 5, "storeRelease: Release object key: %s\n", e->key);
2169 else
2170 debug(20, 5, "storeRelease: Release anonymous object\n");
090089c4 2171
2172 if (e->swap_status == SWAP_OK && (e->swap_file_number > -1)) {
30a4f2a8 2173 (void) safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
090089c4 2174 file_map_bit_reset(e->swap_file_number);
2175 e->swap_file_number = -1;
2176 store_swap_size -= (e->object_len + 1023) >> 10;
a528eb87 2177 HTTPCacheInfo->proto_purgeobject(HTTPCacheInfo,
1ac9bc38 2178 urlParseProtocol(e->url),
30a4f2a8 2179 e->object_len);
090089c4 2180 }
f7a5493b 2181 storeHashDelete(e);
147d3115 2182 storeLog(STORE_LOG_RELEASE, e);
30a4f2a8 2183 destroy_StoreEntry(e);
090089c4 2184 return 0;
2185}
2186
2187
090089c4 2188/* return if the current key is the original one. */
b8d8561b 2189int
2190storeOriginalKey(StoreEntry * e)
090089c4 2191{
2192 if (!e)
2193 return 1;
090089c4 2194 return !(e->flag & KEY_CHANGE);
2195}
2196
2197/* return 1 if a store entry is locked */
24382924 2198static int
b8d8561b 2199storeEntryLocked(StoreEntry * e)
090089c4 2200{
30a4f2a8 2201 if (e->lock_count)
2202 return 1;
2203 if (e->swap_status == SWAPPING_OUT)
2204 return 1;
2205 if (e->mem_status == SWAPPING_IN)
2206 return 1;
a1e47288 2207 if (e->store_status == STORE_PENDING)
2208 return 1;
30a4f2a8 2209 return 0;
090089c4 2210}
2211
2212/* use this for internal call only */
b8d8561b 2213static int
2214storeCopy(StoreEntry * e, int stateoffset, int maxSize, char *buf, int *size)
090089c4 2215{
2216 int available_to_write = 0;
2217
22e4fa85 2218 available_to_write = e->mem_obj->e_current_len - stateoffset;
090089c4 2219
22e4fa85 2220 if (stateoffset < e->mem_obj->e_lowest_offset) {
090089c4 2221 /* this should not happen. Logic race !!! */
c943f331 2222 debug(20, 1, "storeCopy: Client Request a chunk of data in area lower than the lowest_offset\n");
2223 debug(20, 1, " Current Lowest offset : %d\n", e->mem_obj->e_lowest_offset);
2224 debug(20, 1, " Requested offset : %d\n", stateoffset);
090089c4 2225 /* can't really do anything here. Client may hang until lifetime runout. */
2226 return 0;
2227 }
2228 *size = (available_to_write >= maxSize) ?
2229 maxSize : available_to_write;
2230
c943f331 2231 debug(20, 6, "storeCopy: avail_to_write=%d, store_offset=%d\n",
090089c4 2232 *size, stateoffset);
2233
2234 if (*size > 0)
22e4fa85 2235 (void) e->mem_obj->data->mem_copy(e->mem_obj->data, stateoffset, buf, *size);
090089c4 2236
2237 return *size;
2238}
2239
2240/* check if there is any client waiting for this object at all */
2241/* return 1 if there is at least one client */
b8d8561b 2242int
2243storeClientWaiting(StoreEntry * e)
090089c4 2244{
2245 int i;
6e40f263 2246 MemObject *mem = e->mem_obj;
2247 if (mem->client_list) {
2248 for (i = 0; i < mem->client_list_size; i++) {
2249 if (mem->client_list[i])
090089c4 2250 return 1;
2251 }
2252 }
6e40f263 2253 if (mem->pending) {
2254 for (i = 0; i < (int) mem->pending_list_size; i++) {
2255 if (mem->pending[i])
090089c4 2256 return 1;
2257 }
2258 }
2259 return 0;
2260}
2261
2262/* return index to matched clientstatus in client_list, -1 on NOT_FOUND */
b8d8561b 2263static int
2264storeClientListSearch(MemObject * mem, int fd)
090089c4 2265{
2266 int i;
6e40f263 2267 if (mem->client_list) {
2268 for (i = 0; i < mem->client_list_size; i++) {
2269 if (mem->client_list[i] == NULL)
2270 continue;
2271 if (mem->client_list[i]->fd != fd)
2272 continue;
090089c4 2273 return i;
6e40f263 2274 }
090089c4 2275 }
2276 return -1;
2277}
2278
2279/* add client with fd to client list */
24382924 2280static void
b8d8561b 2281storeClientListAdd(StoreEntry * e, int fd, int last_offset)
090089c4 2282{
2283 int i;
6e40f263 2284 MemObject *mem = e->mem_obj;
2285 ClientStatusEntry **oldlist = NULL;
2286 int oldsize;
090089c4 2287 /* look for empty slot */
6e40f263 2288 if (mem->client_list == NULL) {
2289 mem->client_list_size = MIN_CLIENT;
2290 mem->client_list =
2291 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
090089c4 2292 }
6e40f263 2293 for (i = 0; i < mem->client_list_size; i++) {
2294 if (mem->client_list[i] == NULL)
2295 break;
2296 }
2297 if (i == mem->client_list_size) {
2298 debug(20, 3, "storeClientListAdd: FD %d Growing client_list for '%s'\n",
2299 fd, e->url);
2300 oldlist = mem->client_list;
2301 oldsize = mem->client_list_size;
2302 mem->client_list_size <<= 1;
2303 mem->client_list =
2304 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
2305 for (i = 0; i < oldsize; i++)
2306 mem->client_list[i] = oldlist[i];
2307 safe_free(oldlist);
2308 }
2309 mem->client_list[i] = xcalloc(1, sizeof(ClientStatusEntry));
2310 mem->client_list[i]->fd = fd;
2311 mem->client_list[i]->last_offset = last_offset;
090089c4 2312}
2313
2314/* same to storeCopy but also register client fd and last requested offset
2315 * for each client */
b8d8561b 2316int
2317storeClientCopy(StoreEntry * e, int stateoffset, int maxSize, char *buf, int *size, int fd)
090089c4 2318{
2319 int next_offset;
2320 int client_list_index;
22e4fa85 2321 int available_to_write = e->mem_obj->e_current_len - stateoffset;
6e40f263 2322 MemObject *mem = e->mem_obj;
090089c4 2323
6e40f263 2324 if (stateoffset < mem->e_lowest_offset) {
090089c4 2325 /* this should not happen. Logic race !!! */
c943f331 2326 debug(20, 1, "storeClientCopy: Client Request a chunk of data in area lower than the lowest_offset\n");
2327 debug(20, 1, " fd : %d\n", fd);
6e40f263 2328 debug(20, 1, " Current Lowest offset : %d\n", mem->e_lowest_offset);
c943f331 2329 debug(20, 1, " Requested offset : %d\n", stateoffset);
090089c4 2330 /* can't really do anything here. Client may hang until lifetime runout. */
2331 return 0;
2332 }
6e40f263 2333 *size = (available_to_write >= maxSize) ? maxSize : available_to_write;
090089c4 2334
c943f331 2335 debug(20, 6, "storeCopy: avail_to_write=%d, store_offset=%d\n",
090089c4 2336 *size, stateoffset);
2337
2338 /* update the lowest requested offset */
2339 next_offset = (stateoffset + *size);
6e40f263 2340 if ((client_list_index = storeClientListSearch(mem, fd)) >= 0) {
2341 mem->client_list[client_list_index]->last_offset = next_offset;
090089c4 2342 } else {
2343 storeClientListAdd(e, fd, next_offset);
2344 }
2345
2346 if (*size > 0)
6e40f263 2347 (void) mem->data->mem_copy(mem->data, stateoffset, buf, *size);
090089c4 2348
2349 /* see if we can get rid of some data if we are in "delete behind" mode . */
aadcfbad 2350 if (BIT_TEST(e->flag, DELETE_BEHIND)) {
090089c4 2351 /* call the handler to delete behind the lowest offset */
2352 storeDeleteBehind(e);
2353 }
2354 return *size;
2355}
2356
090089c4 2357
b8d8561b 2358int
2359storeEntryValidToSend(StoreEntry * e)
090089c4 2360{
b8de7ebe 2361 if (squid_curtime < e->expires)
9174e204 2362 return 1;
1c481e00 2363 if (e->expires != 0)
2364 return 0; /* Expired! */
2365 if (e->store_status != STORE_PENDING)
2366 return 0;
2367 if (e->mem_status != IN_MEMORY)
2368 return 0;
2369 return 1; /* STORE_PENDING, IN_MEMORY, exp=0 */
090089c4 2370}
2371
24382924 2372static int
b8d8561b 2373storeEntryValidLength(StoreEntry * e)
6602e70e 2374{
ffe4a367 2375 int diff;
2285407f 2376 int hdr_sz;
2377 int content_length;
ffe4a367 2378
2285407f 2379 if (e->mem_obj == NULL)
ffe4a367 2380 fatal_dump("storeEntryValidLength: NULL mem_obj");
2381
2285407f 2382 hdr_sz = e->mem_obj->reply->hdr_sz;
147d3115 2383 content_length = e->mem_obj->reply->content_length;
2285407f 2384
ffe4a367 2385 debug(20, 3, "storeEntryValidLength: Checking '%s'\n", e->key);
60bf30cb 2386 debug(20, 5, "storeEntryValidLength: object_len = %d\n", e->object_len);
2285407f 2387 debug(20, 5, "storeEntryValidLength: hdr_sz = %d\n", hdr_sz);
2388 debug(20, 5, "storeEntryValidLength: content_length = %d\n", content_length);
ffe4a367 2389
2285407f 2390 if (content_length == 0) {
60bf30cb 2391 debug(20, 5, "storeEntryValidLength: Zero content length; assume valid; '%s'\n",
ffe4a367 2392 e->key);
2393 return 1;
2394 }
2285407f 2395 if (hdr_sz == 0) {
60bf30cb 2396 debug(20, 5, "storeEntryValidLength: Zero header size; assume valid; '%s'\n",
ffe4a367 2397 e->key);
2398 return 1;
2399 }
2285407f 2400 diff = hdr_sz + content_length - e->object_len;
ffe4a367 2401 if (diff != 0) {
60bf30cb 2402 debug(20, 3, "storeEntryValidLength: %d bytes too %s; '%s'\n",
ffe4a367 2403 diff < 0 ? -diff : diff,
2404 diff < 0 ? "small" : "big",
2405 e->key);
2406 return 0;
2407 }
2408 return 1;
2409}
6602e70e 2410
b8d8561b 2411static int
2412storeVerifySwapDirs(int clean)
090089c4 2413{
090089c4 2414 int inx;
2415 char *path = NULL;
c943f331 2416 struct stat sb;
2417 int directory_created = 0;
2418 char *cmdbuf = NULL;
090089c4 2419
6e40f263 2420 for (inx = 0; inx < ncache_dirs; inx++) {
090089c4 2421 path = swappath(inx);
9dfb6c1c 2422 debug(20, 9, "storeVerifySwapDirs: Creating swap space in %s\n", path);
090089c4 2423 if (stat(path, &sb) < 0) {
2424 /* we need to create a directory for swap file here. */
2425 if (mkdir(path, 0777) < 0) {
c943f331 2426 if (errno != EEXIST) {
2427 sprintf(tmp_error_buf, "Failed to create swap directory %s: %s",
2428 path,
2429 xstrerror());
2430 fatal(tmp_error_buf);
2431 }
090089c4 2432 }
2433 if (stat(path, &sb) < 0) {
c943f331 2434 sprintf(tmp_error_buf,
2435 "Failed to verify swap directory %s: %s",
090089c4 2436 path, xstrerror());
c943f331 2437 fatal(tmp_error_buf);
090089c4 2438 }
b37a37f9 2439 debug(20, 1, "storeVerifySwapDirs: Created swap directory %s\n", path);
090089c4 2440 directory_created = 1;
2441 }
234967c9 2442 if (clean && opt_unlink_on_reload) {
b37a37f9 2443 debug(20, 1, "storeVerifySwapDirs: Zapping all objects on disk storage.\n");
c943f331 2444 /* This could be dangerous, second copy of cache can destroy
2445 * the existing swap files of the previous cache. We may
2446 * use rc file do it. */
2447 cmdbuf = xcalloc(1, BUFSIZ);
2448 sprintf(cmdbuf, "cd %s; /bin/rm -rf log [0-9][0-9]", path);
b37a37f9 2449 debug(20, 1, "storeVerifySwapDirs: Running '%s'\n", cmdbuf);
c943f331 2450 system(cmdbuf); /* XXX should avoid system(3) */
2451 xfree(cmdbuf);
090089c4 2452 }
2453 }
c943f331 2454 return directory_created;
2455}
090089c4 2456
b8d8561b 2457static void
2458storeCreateSwapSubDirs(void)
c943f331 2459{
f5469bce 2460 int i, j, k;
95d659f0 2461 LOCAL_ARRAY(char, name, MAXPATHLEN);
c943f331 2462 for (j = 0; j < ncache_dirs; j++) {
f5469bce 2463 for (i = 0; i < SWAP_DIRECTORIES_L1; i++) {
2464 sprintf(name, "%s/%02X", swappath(j), i);
e6f18b4a 2465 debug(20, 1, "Making directories in %s\n", name);
c943f331 2466 if (mkdir(name, 0755) < 0) {
2467 if (errno != EEXIST) {
2468 sprintf(tmp_error_buf,
2469 "Failed to make swap directory %s: %s",
2470 name, xstrerror());
2471 fatal(tmp_error_buf);
2472 }
2473 }
f5469bce 2474 for (k = 0; k < SWAP_DIRECTORIES_L2; k++) {
2475 sprintf(name, "%s/%02X/%02X", swappath(j), i, k);
2476 if (mkdir(name, 0755) < 0) {
2477 if (errno != EEXIST) {
2478 sprintf(tmp_error_buf,
2479 "Failed to make swap directory %s: %s",
2480 name, xstrerror());
2481 fatal(tmp_error_buf);
2482 }
2483 }
2484 }
c943f331 2485 }
090089c4 2486 }
c943f331 2487}
090089c4 2488
b8d8561b 2489void
2490storeInit(void)
c943f331 2491{
aadcfbad 2492 int dir_created = 0;
0ffd22bc 2493 wordlist *w = NULL;
234967c9 2494 char *fname = NULL;
aadcfbad 2495 file_map_create(MAX_SWAP_FILE);
2496 storeCreateHashTable(urlcmp);
b6f794d6 2497 if (strcmp((fname = Config.Log.store), "none") == 0)
234967c9 2498 storelog_fd = -1;
2499 else
e47afcdb 2500 storelog_fd = file_open(fname, NULL, O_WRONLY | O_CREAT);
234967c9 2501 if (storelog_fd < 0)
2502 debug(20, 1, "Store logging disabled\n");
b6f794d6 2503 for (w = Config.cache_dirs; w; w = w->next)
0ffd22bc 2504 storeAddSwapDisk(w->key);
a26bdc75 2505 storeSanityCheck();
b6f794d6 2506 dir_created = storeVerifySwapDirs(opt_zap_disk_store);
c943f331 2507 sprintf(swaplog_file, "%s/log", swappath(0));
e47afcdb 2508 swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
66a91d2d 2509 debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, swaplog_file);
c943f331 2510 if (swaplog_fd < 0) {
2511 sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file);
2512 fatal(tmp_error_buf);
2513 }
c943f331 2514 swaplog_lock = file_write_lock(swaplog_fd);
b6f794d6 2515 if (!opt_zap_disk_store)
3facc31e 2516 storeStartRebuildFromDisk();
30a4f2a8 2517 else
2518 store_rebuilding = STORE_NOT_REBUILDING;
b6f794d6 2519 if (dir_created || opt_zap_disk_store)
c943f331 2520 storeCreateSwapSubDirs();
b1c0cc67 2521}
c943f331 2522
b8d8561b 2523void
2524storeConfigure(void)
b1c0cc67 2525{
e954773d 2526 int store_mem_high = 0;
2527 int store_mem_low = 0;
b6f794d6 2528 store_mem_high = (long) (Config.Mem.maxSize / 100) *
2529 Config.Mem.highWaterMark;
2530 store_mem_low = (long) (Config.Mem.maxSize / 100) *
2531 Config.Mem.lowWaterMark;
090089c4 2532
b1c0cc67 2533 store_swap_high = (long) (((float) Config.Swap.maxSize *
2534 (float) Config.Swap.highWaterMark) / (float) 100);
2535 store_swap_low = (long) (((float) Config.Swap.maxSize *
2536 (float) Config.Swap.lowWaterMark) / (float) 100);
e954773d 2537
2538 store_pages_high = store_mem_high / SM_PAGE_SIZE;
2539 store_pages_low = store_mem_low / SM_PAGE_SIZE;
090089c4 2540}
2541
2542/*
2543 * storeSanityCheck - verify that all swap storage areas exist, and
2544 * are writable; otherwise, force -z.
2545 */
24382924 2546static void
b8d8561b 2547storeSanityCheck(void)
090089c4 2548{
95d659f0 2549 LOCAL_ARRAY(char, name, 4096);
090089c4 2550 int i;
2551
2552 if (ncache_dirs < 1)
f133a434 2553 storeAddSwapDisk(DefaultSwapDir);
090089c4 2554
0be3ec60 2555 for (i = 0; i < SWAP_DIRECTORIES_L1; i++) {
2556 sprintf(name, "%s/%02X", swappath(i), i);
090089c4 2557 errno = 0;
2558 if (access(name, W_OK)) {
2559 /* A very annoying problem occurs when access() fails because
b8de7ebe 2560 * the system file table is full. To prevent squid from
090089c4 2561 * deleting your entire disk cache on a whim, insist that the
2562 * errno indicates that the directory doesn't exist */
2563 if (errno != ENOENT)
2564 continue;
b8de7ebe 2565 debug(20, 0, "WARNING: Cannot write to swap directory '%s'\n",
2566 name);
2567 debug(20, 0, "Forcing a *full restart* (e.g., %s -z)...\n",
2568 appname);
b6f794d6 2569 opt_zap_disk_store = 1;
090089c4 2570 return;
2571 }
2572 }
2573}
2574
b8d8561b 2575int
2576urlcmp(char *url1, char *url2)
090089c4 2577{
2578 if (!url1 || !url2)
6eb42cae 2579 fatal_dump("urlcmp: Got a NULL url pointer.");
090089c4 2580 return (strcmp(url1, url2));
2581}
2582
090089c4 2583/*
2584 * This routine is to be called by main loop in main.c.
2585 * It removes expired objects on only one bucket for each time called.
2586 * returns the number of objects removed
2587 *
2588 * This should get called 1/s from main().
2589 */
b8d8561b 2590int
2591storeMaintainSwapSpace(void)
090089c4 2592{
30a4f2a8 2593 static time_t last_time = 0;
090089c4 2594 static unsigned int bucket = 0;
2595 hash_link *link_ptr = NULL, *next = NULL;
2596 StoreEntry *e = NULL;
2597 int rm_obj = 0;
2598
30a4f2a8 2599 /* We can't delete objects while rebuilding swap */
2600 if (store_rebuilding == STORE_REBUILDING_FAST)
2601 return -1;
2602
090089c4 2603 /* Purges expired objects, check one bucket on each calling */
30a4f2a8 2604 if (squid_curtime - last_time >= STORE_MAINTAIN_RATE) {
2605 last_time = squid_curtime;
090089c4 2606 if (bucket >= STORE_BUCKETS)
2607 bucket = 0;
ec74cf5f 2608 link_ptr = hash_get_bucket(store_table, bucket++);
ff5731b1 2609 for (; link_ptr; link_ptr = next) {
090089c4 2610 next = link_ptr->next;
2611 e = (StoreEntry *) link_ptr;
79b5cc5f 2612 if (!storeCheckExpired(e))
2613 continue;
2614 storeRelease(e);
2615 ++rm_obj;
090089c4 2616 }
2617 }
30a4f2a8 2618 debug(20, rm_obj ? 2 : 3, "Removed %d expired objects\n", rm_obj);
79b5cc5f 2619
2620 /* Scan row of hash table each second and free storage if we're
2621 * over the high-water mark */
2622 storeGetSwapSpace(0);
2623
090089c4 2624 return rm_obj;
2625}
2626
090089c4 2627
2628/*
2629 * storeWriteCleanLog
2630 *
2631 * Writes a "clean" swap log file from in-memory metadata.
2632 */
b8d8561b 2633int
2634storeWriteCleanLog(void)
090089c4 2635{
2636 StoreEntry *e = NULL;
95d659f0 2637 LOCAL_ARRAY(char, swapfilename, MAX_FILE_NAME_LEN);
090089c4 2638 FILE *fp = NULL;
2639 int n = 0;
30a4f2a8 2640 int x = 0;
090089c4 2641 time_t start, stop, r;
2642
30a4f2a8 2643 if (store_rebuilding) {
c943f331 2644 debug(20, 1, "storeWriteCleanLog: Not currently OK to rewrite swap log.\n");
2645 debug(20, 1, "storeWriteCleanLog: Operation aborted.\n");
090089c4 2646 return 0;
2647 }
c943f331 2648 debug(20, 1, "storeWriteCleanLog: Starting...\n");
b37a37f9 2649 start = getCurrentTime();
3facc31e 2650 sprintf(tmp_filename, "%s/log_clean", swappath(0));
2651 if ((fp = fopen(tmp_filename, "a+")) == NULL) {
a8f7d3ee 2652 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
090089c4 2653 return 0;
2654 }
2655 for (e = storeGetFirst(); e; e = storeGetNext()) {
090089c4 2656 if (e->swap_file_number < 0)
2657 continue;
2658 if (e->swap_status != SWAP_OK)
2659 continue;
2660 if (e->object_len <= 0)
2661 continue;
3ba50d75 2662 if (BIT_TEST(e->flag, RELEASE_REQUEST))
0a21bd84 2663 continue;
2664 if (BIT_TEST(e->flag, KEY_PRIVATE))
2665 continue;
090089c4 2666 storeSwapFullPath(e->swap_file_number, swapfilename);
f5469bce 2667 x = fprintf(fp, "%08x %08x %08x %08x %9d %s\n",
2668 (int) e->swap_file_number,
2669 (int) e->timestamp,
2670 (int) e->expires,
2671 (int) e->lastmod,
2672 e->object_len,
2673 e->url);
30a4f2a8 2674 if (x < 0) {
a8f7d3ee 2675 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
30a4f2a8 2676 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
2677 fclose(fp);
2678 safeunlink(tmp_filename, 0);
2679 return 0;
2680 }
090089c4 2681 if ((++n & 0xFFF) == 0) {
b37a37f9 2682 getCurrentTime();
c943f331 2683 debug(20, 1, " %7d lines written so far.\n", n);
090089c4 2684 }
2685 }
30a4f2a8 2686 if (fclose(fp) < 0) {
a8f7d3ee 2687 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
30a4f2a8 2688 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
2689 safeunlink(tmp_filename, 0);
2690 return 0;
2691 }
090089c4 2692 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK) {
c943f331 2693 debug(20, 0, "storeWriteCleanLog: Failed to unlock swaplog!\n");
2694 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
090089c4 2695 return 0;
2696 }
3facc31e 2697 if (rename(tmp_filename, swaplog_file) < 0) {
c943f331 2698 debug(20, 0, "storeWriteCleanLog: rename failed: %s\n",
090089c4 2699 xstrerror());
2700 return 0;
2701 }
2702 file_close(swaplog_fd);
e47afcdb 2703 swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
090089c4 2704 if (swaplog_fd < 0) {
c943f331 2705 sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file);
2706 fatal(tmp_error_buf);
090089c4 2707 }
090089c4 2708 swaplog_lock = file_write_lock(swaplog_fd);
2709
b37a37f9 2710 stop = getCurrentTime();
090089c4 2711 r = stop - start;
c943f331 2712 debug(20, 1, " Finished. Wrote %d lines.\n", n);
2713 debug(20, 1, " Took %d seconds (%6.1lf lines/sec).\n",
090089c4 2714 r > 0 ? r : 0, (double) n / (r > 0 ? r : 1));
2715
2716 /* touch a timestamp file */
3facc31e 2717 sprintf(tmp_filename, "%s/log-last-clean", swappath(0));
2718 file_close(file_open(tmp_filename, NULL, O_WRONLY | O_CREAT | O_TRUNC));
090089c4 2719 return n;
2720}
2721
24382924 2722static int
b8d8561b 2723swapInError(int fd_unused, StoreEntry * entry)
090089c4 2724{
b8de7ebe 2725 squid_error_entry(entry, ERR_DISK_IO, xstrerror());
090089c4 2726 return 0;
2727}
2728
b8d8561b 2729int
2730storePendingNClients(StoreEntry * e)
090089c4 2731{
2732 int npend = 0;
2733 int i;
2734
2735 if (!e->mem_obj)
2736 return 0;
22e4fa85 2737 for (npend = i = 0; i < (int) e->mem_obj->pending_list_size; i++) {
2738 if (e->mem_obj->pending[i])
090089c4 2739 npend++;
2740 }
2741 return npend;
2742}
d8b45066 2743
b8d8561b 2744void
2745storeRotateLog(void)
d8b45066 2746{
2747 char *fname = NULL;
2748 int i;
95d659f0 2749 LOCAL_ARRAY(char, from, MAXPATHLEN);
2750 LOCAL_ARRAY(char, to, MAXPATHLEN);
d8b45066 2751
234967c9 2752 if (storelog_fd > -1) {
2753 file_close(storelog_fd);
2754 storelog_fd = -1;
2755 }
b6f794d6 2756 if ((fname = Config.Log.store) == NULL)
2d25dc1a 2757 return;
d8b45066 2758
234967c9 2759 if (strcmp(fname, "none") == 0)
2760 return;
2761
d8b45066 2762 debug(20, 1, "storeRotateLog: Rotating.\n");
2763
2764 /* Rotate numbers 0 through N up one */
b6f794d6 2765 for (i = Config.Log.rotateNumber; i > 1;) {
2d25dc1a 2766 i--;
2767 sprintf(from, "%s.%d", fname, i - 1);
2768 sprintf(to, "%s.%d", fname, i);
2769 rename(from, to);
d8b45066 2770 }
2771 /* Rotate the current log to .0 */
b6f794d6 2772 if (Config.Log.rotateNumber > 0) {
2d25dc1a 2773 sprintf(to, "%s.%d", fname, 0);
2774 rename(fname, to);
d8b45066 2775 }
e47afcdb 2776 storelog_fd = file_open(fname, NULL, O_WRONLY | O_CREAT);
234967c9 2777 if (storelog_fd < 0) {
2778 debug(20, 0, "storeRotateLog: %s: %s\n", fname, xstrerror());
2779 debug(20, 1, "Store logging disabled\n");
2780 }
d8b45066 2781}
30a4f2a8 2782
56f29785 2783static int
9d90e665 2784storeShouldPurgeMem(StoreEntry * e)
56f29785 2785{
2786 if (storeCheckPurgeMem(e) == 0)
2787 return 0;
e954773d 2788 if (sm_stats.n_pages_in_use > store_pages_low)
56f29785 2789 return 1;
2790 return 0;
2791}
2792
2793
30a4f2a8 2794/*
2795 * Check if its okay to remove the memory data for this object, but
2796 * leave the StoreEntry around. Designed to be called from
2797 * storeUnlockObject() and storeSwapOutHandle().
2798 */
b8d8561b 2799static int
2800storeCheckPurgeMem(StoreEntry * e)
30a4f2a8 2801{
2802 if (storeEntryLocked(e))
2803 return 0;
2804 if (e->store_status != STORE_OK)
2805 return 0;
3e98df20 2806 if (e->swap_status != SWAP_OK)
2807 return 0;
30a4f2a8 2808 return 1;
2809}
a1e47288 2810
b8d8561b 2811static int
2812storeCheckExpired(StoreEntry * e)
620da955 2813{
2814 if (storeEntryLocked(e))
2815 return 0;
79b5cc5f 2816 if (squid_curtime - e->expires < Config.expireAge)
620da955 2817 return 0;
2818 return 1;
2819}
2820
b8d8561b 2821static char *
2822storeDescribeStatus(StoreEntry * e)
a1e47288 2823{
2824 static char buf[MAX_URL << 1];
2825 sprintf(buf, "mem:%13s ping:%12s store:%13s swap:%12s locks:%d %s\n",
2826 memStatusStr[e->mem_status],
2827 pingStatusStr[e->ping_status],
2828 storeStatusStr[e->store_status],
2829 swapStatusStr[e->swap_status],
2830 (int) e->lock_count,
2831 e->url);
2832 return buf;
2833}
457bedbd 2834
b8d8561b 2835void
2836storeCloseLog(void)
457bedbd 2837{
2838 file_close(swaplog_fd);
2839 file_close(storelog_fd);
2840}
79b5cc5f 2841
b8d8561b 2842void
2843storeNegativeCache(StoreEntry * e)
79b5cc5f 2844{
ff5731b1 2845 e->expires = squid_curtime + Config.negativeTtl;
2846 BIT_SET(e->flag, ENTRY_NEGCACHED);
79b5cc5f 2847}
0a21bd84 2848
2849void
2850storeFreeMemory(void)
2851{
2852 StoreEntry *e;
df92bd2a 2853 StoreEntry **list;
2854 int i = 0;
ea6b05e3 2855 int j;
56e15c50 2856 list = xcalloc(meta_data.store_entries, sizeof(StoreEntry *));
df92bd2a 2857 e = (StoreEntry *) hash_first(store_table);
2858 while (e && i < meta_data.store_entries) {
2859 *(list + i) = e;
2860 i++;
2861 e = (StoreEntry *) hash_next(store_table);
2862 }
ea6b05e3 2863 for (j = 0; j < i; j++)
2864 destroy_StoreEntry(*(list + j));
df92bd2a 2865 xfree(list);
0a21bd84 2866 hashFreeMemory(store_table);
2867}
Mirror of https://github.com/squid-cache/squid.git