From: kostas <> Date: Sat, 10 Jan 1998 14:50:33 +0000 (+0000) Subject: At last, added meta-data to swap files. Still testing X-Git-Tag: SQUID_3_0_PRE1~4216 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a21fbb54f78b59f0bdd2a87e17534d046c8894ae;p=thirdparty%2Fsquid.git At last, added meta-data to swap files. Still testing --- diff --git a/src/defines.h b/src/defines.h index b6eb508237..6a006812a0 100644 --- a/src/defines.h +++ b/src/defines.h @@ -173,3 +173,7 @@ #define VIEWINCLUDED 1 #define VIEWEXCLUDED 2 #endif + +#define META_OK 0x03 +#define META_DIRTY 0x04 +#define META_BAD 0x05 diff --git a/src/enums.h b/src/enums.h index 78f0dee6bf..3cf2aa0b8c 100644 --- a/src/enums.h +++ b/src/enums.h @@ -374,3 +374,14 @@ enum { SNMP_C_USER, SNMP_C_COMMUNITY }; + +enum { + SWAP_META_VOID, /* should not come up */ + SWAP_META_KEY_URL, /* key w/ keytype */ + SWAP_META_KEY_SHA, + SWAP_META_KEY_MD5, + SWAP_META_URL, /* the url , if not in the header */ + SWAP_META_STD, /* standard metadata */ + SWAP_META_HITMETERING, /* reserved for hit metering */ + SWAP_META_VALID +}; diff --git a/src/globals.h b/src/globals.h index d65dc42581..07943343c2 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.24 1998/01/06 07:11:53 wessels Exp $ + * $Id: globals.h,v 1.25 1998/01/10 07:50:35 kostas Exp $ */ extern FILE *debug_log; /* NULL */ @@ -56,6 +56,7 @@ extern int icmp_sock; /* -1 */ extern int neighbors_do_private_keys; /* 1 */ extern int opt_accel_uses_host; /* 0 */ extern int opt_catch_signals; /* 1 */ +extern int opt_convert; /* 0 */ extern int opt_debug_stderr; /* 0 */ extern int opt_dns_tests; /* 1 */ extern int opt_foreground_rebuild; /* 0 */ diff --git a/src/main.cc b/src/main.cc index bce6b8b516..c2d05fa909 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,6 @@ /* - * $Id: main.cc,v 1.205 1998/01/07 21:16:31 wessels Exp $ + * $Id: main.cc,v 1.206 1998/01/10 07:50:36 kostas Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -149,6 +149,7 @@ usage(void) " -v Print version.\n" " -z Create swap directories\n" " -C Do not catch fatal signals.\n" + " -c Convert store from <1.2.\n" " -D Disable initial DNS tests.\n" " -F Foreground fast store rebuild.\n" " -N No daemon mode.\n" @@ -166,7 +167,7 @@ mainParseOptions(int argc, char *argv[]) extern char *optarg; int c; - while ((c = getopt(argc, argv, "CDFNRVYXa:df:hk:m:su:vz?")) != -1) { + while ((c = getopt(argc, argv, "CDFNRVYXa:cdf:hk:m:su:vz?")) != -1) { switch (c) { case 'C': opt_catch_signals = 0; @@ -196,6 +197,9 @@ mainParseOptions(int argc, char *argv[]) case 'a': httpPortNumOverride = atoi(optarg); break; + case 'c': + opt_convert = 1; + break; case 'd': opt_debug_stderr = 1; break; @@ -358,7 +362,9 @@ mainReconfigure(void) dnsShutdownServers(); asnCleanup(); redirectShutdownServers(); +#if 0 storeDirCloseSwapLogs(); +#endif errorFree(); parseConfigFile(ConfigFile); _db_init(Config.Log.log, Config.debugOptions); @@ -371,7 +377,9 @@ mainReconfigure(void) serverConnectionsOpen(); if (theOutIcpConnection >= 0 && (!Config2.Accel.on || Config.onoff.accel_with_proxy)) neighbors_open(theOutIcpConnection); +#if 0 storeDirOpenSwapLogs(); +#endif debug(1, 0) ("Ready to serve requests.\n"); } diff --git a/src/store.cc b/src/store.cc index 47d16cfb32..7172cd4f4d 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.364 1998/01/06 07:11:55 wessels Exp $ + * $Id: store.cc,v 1.365 1998/01/10 07:50:38 kostas Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -120,6 +120,26 @@ #define STORE_LOG_SWAPOUT 2 #define STORE_LOG_RELEASE 3 +#if STORE_KEY_SHA +#define SWAP_META_KEY SWAP_META_KEY_SHA +#define squid_key_size SHA_DIGEST_INTS*sizeof(int) +#elif STORE_KEY_MD5 +#define SWAP_META_KEY SWAP_META_KEY_MD5 +#define squid_key_size MD5_DIGEST_CHARS +#else +#define SWAP_META_KEY SWAP_META_KEY_URL +#define squid_key_size -1 +#endif + + +#include +#define SWAP_META_TLD_START sizeof(int)+sizeof(char) +#define SWAP_META_TLD_SIZE SWAP_META_TLD_START +#define SwapMetaType(x) (char)x[0] +#define SwapMetaSize(x) &x[sizeof(char)] +#define SwapMetaData(x) &x[SWAP_META_TLD_START] +#define HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(int)) + static char *storeLogTags[] = { "CREATE", @@ -242,7 +262,7 @@ static void storeStartRebuildFromDisk(void); static void storeSwapOutStart(StoreEntry * e); static DWCB storeSwapOutHandle; static void storeSetPrivateKey(StoreEntry *); -static EVH storeDoRebuildFromDisk; +static EVH storeDoConvertFromLog; static EVH storeCleanup; static VCB storeCleanupComplete; static void storeValidate(StoreEntry *, VCB *, void *); @@ -264,6 +284,23 @@ static void storeClientCopy2(StoreEntry *, store_client *); static void storeHashInsert(StoreEntry * e, const cache_key *); static void storeSwapOutFileClose(StoreEntry * e); +/* functions implementing meta data on store */ +static void storeConvert(void); +static void +storeConvertFile(const cache_key *,int,int,time_t,time_t,time_t,time_t, + u_num32, u_num32, int); + +static int storeBuildMetaData(StoreEntry *, char *); +static int storeGetMetaBuf(const char *, MemObject *); +#if 0 +static int storeParseMetaBuf(StoreEntry *); +#endif +static int storeGetNextFile(int *sfileno, int *size); +static void addSwapHdr(int, int, void *, char *, int *); +static int getSwapHdr(int *, int *, void *, char *, int); +static EVH storeDoRebuildFromSwapFiles; + + /* Now, this table is inaccessible to outsider. They have to use a method * to access a value in internal storage data structure. */ static hash_table *store_table = NULL; @@ -322,6 +359,7 @@ destroy_MemObject(MemObject * mem) destroy_MemObjectData(mem); meta_data.misc -= strlen(mem->log_url); assert(mem->clients == NULL); + safe_free(mem->swapout.meta_buf); safe_free(mem->reply); safe_free(mem->url); safe_free(mem->log_url); @@ -800,7 +838,7 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) return; } mem->swapout.done_offset += len; - if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len) { + if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len + mem->swapout.meta_len ) { storeCheckSwapOut(e); return; } @@ -815,7 +853,9 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) FALSE); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); +#if 0 storeDirSwapLog(e); +#endif } /* Note, we don't otherwise call storeReleaseRequest() here because * storeCheckCachable() does it for is if necessary */ @@ -832,6 +872,7 @@ storeCheckSwapOut(StoreEntry * e) char *swap_buf; ssize_t swap_buf_len; int x; + int hdr_len=0; assert(mem != NULL); /* should we swap something out to disk? */ debug(20, 3) ("storeCheckSwapOut: %s\n", mem->url); @@ -888,13 +929,19 @@ storeCheckSwapOut(StoreEntry * e) if (e->swap_status == SWAPOUT_OPENING) return; assert(mem->swapout.fd > -1); - if (swapout_size > SWAP_BUF) - swapout_size = SWAP_BUF; + swap_buf = get_free_8k_page(); + if (mem->swapout.queue_offset==0) + hdr_len= storeBuildMetaData(e, swap_buf); + + if (swapout_size > SWAP_BUF - hdr_len) + swapout_size = SWAP_BUF - hdr_len; + swap_buf_len = memCopy(mem->data, mem->swapout.queue_offset, - swap_buf, - swapout_size); + swap_buf+hdr_len, + swapout_size) + hdr_len; + if (swap_buf_len < 0) { debug(20, 1) ("memCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); /* XXX This is probably wrong--we should storeRelease()? */ @@ -910,7 +957,7 @@ storeCheckSwapOut(StoreEntry * e) assert(swap_buf_len > 0); debug(20, 3) ("storeCheckSwapOut: swapping out %d bytes from %d\n", swap_buf_len, mem->swapout.queue_offset); - mem->swapout.queue_offset += swap_buf_len; + mem->swapout.queue_offset += swap_buf_len-hdr_len; x = file_write(mem->swapout.fd, swap_buf, swap_buf_len, @@ -980,6 +1027,9 @@ storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) return; } } + debug(20,3)("storeSwapInStart: called for %08X %s \n", + e->swap_file_number, e->key?e->key:"[no key]"); + assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); assert(e->swap_file_number >= 0); assert(e->mem_obj != NULL); @@ -987,10 +1037,15 @@ storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) ctrlp->e = e; ctrlp->callback = callback; ctrlp->callback_data = callback_data; - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) + if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) { + debug(20,3)("storeSwapInStart: calling storeSwapInValidateComplete GREEN\n"); storeSwapInValidateComplete(ctrlp); - else + } + else { + debug(20,3)("storeSwapInStart: calling storeValidate RED\n"); storeValidate(e, storeSwapInValidateComplete, ctrlp); + + } } @@ -1008,6 +1063,9 @@ storeSwapInValidateComplete(void *data) return; } ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); + + debug(20,3)("storeSwapInValidateComplete: Opening %s\n", ctrlp->path); + file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp); } @@ -1021,7 +1079,8 @@ storeSwapInFileOpened(void *data, int fd) assert(e->mem_status == NOT_IN_MEMORY); assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); if (fd < 0) { - debug(20, 0) ("storeSwapInStartComplete: Failed for '%s'\n", mem->url); + debug(20, 0) ("storeSwapInStartComplete: Failed for '%s' (%s)\n", mem->url, + ctrlp->path); /* Invoke a store abort that should free the memory object */ (ctrlp->callback) (-1, ctrlp->callback_data); xfree(ctrlp->path); @@ -1035,10 +1094,10 @@ storeSwapInFileOpened(void *data, int fd) xfree(ctrlp); } -/* recreate meta data from disk image in swap directory */ -/* Add one swap file at a time from disk storage */ +/* convert storage .. this is the old storeDoRebuildFromDisk() */ + static void -storeDoRebuildFromDisk(void *data) +storeDoConvertFromLog(void *data) { struct storeRebuildState *RB = data; LOCAL_ARRAY(char, swapfile, MAXPATHLEN); @@ -1066,7 +1125,9 @@ storeDoRebuildFromDisk(void *data) const cache_key *key; /* load a number of objects per invocation */ + if ((d = RB->rebuild_dir) == NULL) { + debug(20,3)("Done Converting, here are the stats.\n"); storeRebuiltFromDisk(RB); return; } @@ -1078,7 +1139,7 @@ storeDoRebuildFromDisk(void *data) storeDirCloseTmpSwapLog(d->dirn); RB->rebuild_dir = d->next; safe_free(d); - eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); + eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); return; } if ((++RB->linecount & 0x3FFF) == 0) @@ -1135,7 +1196,7 @@ storeDoRebuildFromDisk(void *data) key = storeKeyScan(keytext); if (key == NULL) { - debug(20, 1) ("storeDoRebuildFromDisk: bad key: '%s'\n", keytext); + debug(20, 1) ("storeDoConvertFromLog: bad key: '%s'\n", keytext); continue; } e = storeGet(key); @@ -1189,7 +1250,7 @@ storeDoRebuildFromDisk(void *data) } /* update store_swap_size */ RB->objcount++; - e = storeAddDiskRestore(key, + storeConvertFile(key, sfileno, (int) size, expires, @@ -1199,13 +1260,15 @@ storeDoRebuildFromDisk(void *data) (u_num32) scan6, /* refcount */ (u_num32) scan7, /* flags */ d->clean); +#if 0 storeDirSwapLog(e); +#endif } RB->rebuild_dir = d->next; for (D = &RB->rebuild_dir; *D; D = &(*D)->next); *D = d; d->next = NULL; - eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); + eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); } static void @@ -1356,35 +1419,23 @@ storeRebuiltFromDisk(struct storeRebuildState *data) static void storeStartRebuildFromDisk(void) { - int i; struct storeRebuildState *RB; struct _rebuild_dir *d; - FILE *fp; - int clean; + int clean=1; RB = xcalloc(1, sizeof(struct storeRebuildState)); RB->start = squid_curtime; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - fp = storeDirOpenTmpSwapLog(i, &clean); - if (fp == NULL) - continue; - d = xcalloc(1, sizeof(struct _rebuild_dir)); - d->dirn = i; - d->log = fp; - d->clean = clean; - d->speed = opt_foreground_rebuild ? 1 << 30 : 50; - d->next = RB->rebuild_dir; - RB->rebuild_dir = d; - if (!clean) - RB->need_to_validate = 1; - debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n", - i, clean ? "CLEAN" : "DIRTY"); - } - RB->line_in_sz = 4096; - RB->line_in = xcalloc(1, RB->line_in_sz); + d = xcalloc(1, sizeof(struct _rebuild_dir)); + d->clean = clean; + d->speed = opt_foreground_rebuild ? 1 << 30 : 50; + RB->rebuild_dir = d; + if (!clean) + RB->need_to_validate = 1; + debug(20, 1) ("Rebuilding storage (%s)\n", + clean ? "CLEAN" : "DIRTY"); if (opt_foreground_rebuild) { - storeDoRebuildFromDisk(RB); + storeDoRebuildFromSwapFiles(RB); } else { - eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); + eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); } } @@ -1797,10 +1848,15 @@ storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, voi store_client *sc = data; MemObject *mem = sc->mem; STCB *callback = sc->callback; + int hdr_len=0; assert(sc->callback != NULL); debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); - if (sc->copy_offset == 0 && len > 0 && mem != NULL) + if (sc->copy_offset == 0 && len > 0 && mem != NULL) { + hdr_len=storeGetMetaBuf(buf, mem); + memmove((char *)buf, (char *)(buf+hdr_len) , len - hdr_len); + len-=hdr_len; httpParseReplyHeaders(buf, mem->reply); + } sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, len); } @@ -1907,7 +1963,14 @@ storeInit(void) ERROR_BUF_SZ); fatal(tmp_error_buf); } - storeDirOpenSwapLogs(); + if (opt_convert) { + storeDirOpenSwapLogs(); + storeConvert(); + debug(0,0)("DONE Converting. Welcome to %s!\n", version_string); + storeDirCloseSwapLogs(); + exit(0); + } + storeStartRebuildFromDisk(); all_list.head = all_list.tail = NULL; inmem_list.head = inmem_list.tail = NULL; @@ -2433,3 +2496,447 @@ storeBufferFlush(StoreEntry * e) EBIT_CLR(e->flag, DELAY_SENDING); InvokeHandlers(e); } + + + +static int +storeGetNextFile(int *sfileno,int *size) +{ + static int dirn, curlvl1, curlvl2, flag, done, in_dir,fn; + static struct dirent *entry; + static DIR *td; + int fd = 0, used=0; + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, fullpath, SQUID_MAXPATHLEN); + + + debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", flag, + dirn, curlvl1, curlvl2); + + if (done) + return -2; + + while (!fd && !done) { + fd=0; + if (!flag) { /* initialize, open first file */ + done = dirn = curlvl1 = curlvl2 = in_dir = 0; + flag = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (!in_dir) { /* we need to read in a new directory */ + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[dirn].path, + curlvl1, curlvl2); + if (flag && td) + closedir(td); + td = opendir(fullpath); + entry = readdir(td); /* skip . and .. */ + entry = readdir(td); + if (errno == ENOENT) { + debug(20, 3) ("storeGetNextFile: directory does not exist!.\n"); + } + debug(20,3)("storeGetNextFile: Directory %s/%02X/%02X\n", + Config.cacheSwap.swapDirs[dirn].path, + curlvl1, curlvl2); + } + if ((entry = readdir(td))) { + in_dir++; + if (sscanf(entry->d_name, "%x", sfileno) != 1) { + debug(20, 3) ("storeGetNextFile: invalid %s\n", + entry->d_name); + continue; + } + fn=*sfileno; + fn = storeDirProperFileno(dirn, fn); + *sfileno=fn; + used = storeDirMapBitTest(fn); + if (used) { + debug(20,3)("storeGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", + fullpath, entry->d_name); + debug(20, 3) ("storeGetNextFile: Opening %s\n", fullfilename); + fd = file_open(fullfilename, O_RDONLY , NULL, NULL); + continue; + } +#if 0 + else + if (!in_dir) debug(20, 3) ("storeGetNextFile: empty dir.\n"); +#endif + + in_dir=0; + + if ((curlvl2 = (curlvl2 + 1) % Config.cacheSwap.swapDirs[dirn].l2)) + continue; + if ((curlvl1 = (curlvl1 + 1) % Config.cacheSwap.swapDirs[dirn].l1)) + continue; + if ((dirn = (dirn + 1) % Config.cacheSwap.n_configured)) + continue; + else + done=1; + + } + return fd; +} +static void +storeDoRebuildFromSwapFiles(void *data) +{ + struct storeRebuildState *RB = data; + LOCAL_ARRAY(char, hdr_buf, 2*MAX_URL); + LOCAL_ARRAY(cache_key, keybuf, MAX_URL); + StoreEntry *e = NULL; + StoreEntry tmpe; + int sfileno = 0; + int count; + int size; + int x; + struct _rebuild_dir *d = RB->rebuild_dir; + struct stat fst; + static int filecount; + int hdr_len = 0; + int myt, myl; + int fd = 0; + debug(20, 3) (" Starting StoreRebuildFromSwapFiles at speed %d\n", + d->speed); + + for (count = 0; count < d->speed; count++) { + if (fd) + file_close(fd); + fd = storeGetNextFile(&sfileno,&size); + + switch (fd) { + case 0: + continue; + case -1: + debug(20, 1) (" Problem with rebuilding.\n"); + return; + case -2: + debug(20, 1) ("StoreRebuildFromSwapFiles: done!\n"); + store_rebuilding=0; + return; + default: + } + /* lets get file stats here */ + + x=fstat(fd,&fst); + assert(x==0); + + if ((++filecount & 0x3FFF) == 0) + debug(20, 1) (" %7d objects read so far.\n", RB->linecount); + + debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); + + x=read(fd,hdr_buf , 4096); + if (x x (%d)%d\n", hdr_len,x); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + debug(20, 3) (" header size %d\n", hdr_len); + + /* get key */ + + if (!getSwapHdr(&myt, &myl, keybuf, hdr_buf, hdr_len)) { + debug(20,1)("Error getting SWAP_META_KEY %d\n",x); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + keybuf[myl]=0; + + debug(20, 3) (" hm, we have %s, %d, %d\n", keybuf, myt, myl); + + if (keybuf == '\0' || myt!=SWAP_META_KEY) { + debug(20, 1) ("storeDoRebuildFromSwapFiles: bad key\n"); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + + continue; + } + e = storeGet(keybuf); + + /* get the standard meta data for the StoreEntry */ + + if (!getSwapHdr(&myt, &myl, &(tmpe.timestamp), hdr_buf, hdr_len)) { + debug(20,1)("storeDoRebuildFromSwapFiles:Error getting SWAP_META_STD %d\n",myl); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + + continue; + } + + /* check sizes */ + + if (hdr_len+tmpe.object_len != fst.st_size) { + debug(20,1)("storeDoRebuildFromSwapFiles:INVALID swapfile, sizes dont match %d+%d!=%d\n", + hdr_len, tmpe.object_len, fst.st_size); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + + if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) { + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + RB->badflags++; + continue; + } + + if (e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + RB->dupcount++; + } + /* update store_swap_size */ + RB->objcount++; +debug(20,4)("storeDoRebuildFromSwapFiles: KEY=%20s , sfileno=%08X exp=%08X timest=%08X\n", + keybuf, sfileno, tmpe.expires, tmpe.timestamp); +debug(20,4)(" lastref=%08X lastmod=%08X refcount=%08X flag=%08X\n", + tmpe.lastref,tmpe.lastmod,tmpe.refcount,tmpe.flag); +debug(20,4)(" len=%d hdr_len=%d file_len=%d\n",tmpe.object_len, + hdr_len, fst.st_size); + + e = storeAddDiskRestore(keybuf, + sfileno, + (int) tmpe.object_len, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + (u_num32) tmpe.refcount, /* refcount */ + (u_num32) tmpe.flag, /* flags */ + d->clean); + } + eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); +} + + +/* build swapfile header */ +static int +storeBuildMetaData(StoreEntry * e,char *swap_buf_c) +{ + MemObject *mem; + int keylength; + int a=SWAP_META_TLD_START; + char *meta_buf; + + mem=e->mem_obj; + meta_buf=mem->swapout.meta_buf; + + debug(20, 3) ("storeBuildSwapFileHeader: called.\n"); + assert(e->swap_status == SWAPOUT_WRITING); + + if (!meta_buf) + meta_buf=mem->swapout.meta_buf=xmalloc(1024); + +/* construct header */ + + /* add Length(int)-Type(char)-Data encoded info */ + + if (squid_key_size < 0) + keylength = strlen(e->key); + else + keylength = squid_key_size; + + meta_buf[0]=META_OK; + xmemcpy(&meta_buf[1], &a, sizeof(int)); + mem->swapout.meta_len=SWAP_META_TLD_START; + + addSwapHdr(SWAP_META_KEY, keylength, (void *) e->key, + mem->swapout.meta_buf, &mem->swapout.meta_len); + addSwapHdr(SWAP_META_STD,HDR_METASIZE,(void *)&e->timestamp, + mem->swapout.meta_buf, &mem->swapout.meta_len); + debug(20, 3) ("storeBuildSwapFileHeader: len=%d.\n", mem->swapout.meta_len); + + if (swap_buf_c) + xmemcpy(swap_buf_c, mem->swapout.meta_buf, mem->swapout.meta_len); + return mem->swapout.meta_len; +} + + +static int +getSwapHdr(int *type, int *len, void *dst, char *write_buf, int hdr_len) +{ + static int cur; + static char *curptr; + char *tmp_buf; + + if (!cur || curptr!=write_buf) { /* first call or rewind ! */ + cur = SWAP_META_TLD_START; + curptr=write_buf; + } + + if (cur+SWAP_META_TLD_START>hdr_len) { + debug(20,3)("getSwapHdr: overflow, %d %d.\n",cur,hdr_len); + cur=0; + return -1; + } + + tmp_buf = (char *) &write_buf[cur]; /* position ourselves */ + + xmemcpy(len, SwapMetaSize(tmp_buf),sizeof(int)); /* length */ + *type=SwapMetaType(tmp_buf); /* type */ + xmemcpy(dst, SwapMetaData(tmp_buf), *len); /* data */ + + cur += SWAP_META_TLD_START + *len; /* advance position */ + + debug(20, 4) ("getSwapHdr: t=%d l=%d (cur=%d hdr_len=%d) (%p)\n", + *type, *len, cur, hdr_len, dst); + if (cur==hdr_len) { + debug(20,4)("getSwapHdr: finished with this.\n"); + cur=0; + return 1; + } + + return 1; /* ok ! */ +} + + +static void +addSwapHdr(int type, int len, void *src, char *write_buf, int *write_len) +{ + int hdr_len = *write_len; + char *base=&write_buf[hdr_len]; + debug(20,3) ("addSwapHdr: at %d\n",hdr_len); + + base[0]=(char)type; + xmemcpy(&base[1], &len, sizeof(int)); + xmemcpy(SwapMetaData(base), src, len); + + hdr_len += SWAP_META_TLD_START + len; + + /* now we know length */ + + debug(20, 3) ("addSwapHdr: added type=%d len=%d data=%p. hdr_len=%d\n", + type, len, src, hdr_len); + + /* update header */ + xmemcpy(&write_buf[1], &hdr_len, sizeof(int)); + *write_len=hdr_len; +} + +static int +storeGetMetaBuf(const char *buf, MemObject *mem) +{ + int hdr_len; + + assert(mem!=NULL); + + /* the key */ + if (SwapMetaType(buf) != META_OK) { + debug(20, 1) ("storeGetMetaBuf:Found an old-style object, damn.\n"); + return -1; + } + xmemcpy(&hdr_len , SwapMetaSize(buf), sizeof(int)); + mem->swapout.meta_len=hdr_len; + mem->swapout.meta_buf=xmalloc(hdr_len); + xmemcpy(mem->swapout.meta_buf,buf, hdr_len); + + debug(20, 3) (" header size %d\n", hdr_len); + + return hdr_len; +} + +#if 0 +static int +storeParseMetaBuf(StoreEntry *e) +{ + static char mbuf[1024]; + int myt,myl; + MemObject *mem=e->mem_obj; + + assert(e && e->mem_obj && e->key); + getSwapHdr(&myt,&myl, mbuf , mem->swapout.meta_buf, mem->swapout.meta_len); + mbuf[myl]=0; + debug(20,3)("storeParseMetaBuf: key=%s\n",mbuf); + e->key=xstrdup(storeKeyScan(mbuf)); + getSwapHdr(&myt, &myl, &e->timestamp, mem->swapout.meta_buf, mem->swapout.meta_len); + + return 1; +} +#endif + +static void +storeConvert(void) +{ + int i; + struct storeRebuildState *RB; + struct _rebuild_dir *d; + FILE *fp; + int clean; + RB = xcalloc(1, sizeof(struct storeRebuildState)); + RB->start = squid_curtime; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + fp = storeDirOpenTmpSwapLog(i, &clean); + if (fp == NULL) + continue; + d = xcalloc(1, sizeof(struct _rebuild_dir)); + d->dirn = i; + d->log = fp; + d->clean = clean; + d->speed = 1 << 30; + d->next = RB->rebuild_dir; + RB->rebuild_dir = d; + if (!clean) + RB->need_to_validate = 1; + debug(20, 1) ("Converting storage in Cache Dir #%d (%s)\n", + i, clean ? "CLEAN" : "DIRTY"); + } + RB->line_in_sz = 4096; + RB->line_in = xcalloc(1, RB->line_in_sz); + storeDoConvertFromLog(RB); +} + +static void +storeConvertFile(const cache_key * key, + int file_number, + int size, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean) +{ + int fd_r, fd_w; + int hdr_len,x,y; + LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, copybuf, SWAP_BUF); + StoreEntry e; + e.key=key; + e.object_len=size; + e.expires=expires; + e.lastref=lastref; + e.refcount=refcount; + e.flag=flags; + + + storeSwapFullPath(file_number, swapfilename); + fd_r = open(swapfilename, O_RDONLY); + if (fd_r<0) { /* ERROR */ + + return; + } + safeunlink(swapfilename, 1); + fd_w = open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC); + + hdr_len = storeBuildMetaData(&e, copybuf); + x=write(fd_w, copybuf, hdr_len); + while (x>0) { + y=read(fd_r,copybuf, SWAP_BUF); + x=write(fd_w, copybuf, y); + } + close(fd_r); close(fd_w); +} diff --git a/src/structs.h b/src/structs.h index 96b85ee5ed..1c47a32d91 100644 --- a/src/structs.h +++ b/src/structs.h @@ -842,6 +842,8 @@ struct _MemObject { off_t queue_offset; off_t done_offset; int fd; + int meta_len; + char *meta_buf; } swapout; struct _http_reply *reply; request_t *request; @@ -868,10 +870,11 @@ struct _StoreEntry { time_t expires; time_t lastmod; int object_len; + u_short refcount; + u_short flag; + int swap_file_number; dlink_node lru; - u_short flag; - u_short refcount; u_char lock_count; /* Assume < 256! */ mem_status_t mem_status:3; ping_status_t ping_status:3;