From 69c01cd7243ee648a7b70282c138d0d651cd5329 Mon Sep 17 00:00:00 2001 From: wessels <> Date: Tue, 3 Feb 1998 14:35:25 +0000 Subject: [PATCH] Started working on swap headers and meta data. I changed the Get/Add routines to something which Packs/Unpacks the whole buffer to/from a linked list of TLV structures. The hard part now is that the swapin and swapout routines most certainly need to be modified to have special read/write meta header code. --- src/defines.h | 6 +- src/enums.h | 6 +- src/globals.h | 3 +- src/protos.h | 9 +- src/store.cc | 4 +- src/store_client.cc | 3 + src/store_key_md5.cc | 5 ++ src/store_rebuild.cc | 156 +++++++++++++++++------------------ src/store_swapin.cc | 4 +- src/store_swapmeta.cc | 187 ++++++++++++++++++++---------------------- src/store_swapout.cc | 24 ++++-- src/structs.h | 13 +-- src/typedefs.h | 1 + 13 files changed, 220 insertions(+), 201 deletions(-) diff --git a/src/defines.h b/src/defines.h index 23063686a2..fee7bb3e57 100644 --- a/src/defines.h +++ b/src/defines.h @@ -174,9 +174,9 @@ #define VIEWEXCLUDED 2 #endif -#define META_OK 0x03 -#define META_DIRTY 0x04 -#define META_BAD 0x05 +#define STORE_META_OK 0x03 +#define STORE_META_DIRTY 0x04 +#define STORE_META_BAD 0x05 #define IPC_NONE 0 #define IPC_TCP_SOCKET 1 diff --git a/src/enums.h b/src/enums.h index 1dbff23169..92c49f587c 100644 --- a/src/enums.h +++ b/src/enums.h @@ -449,6 +449,9 @@ typedef enum { MEM_MAX } mem_type; +/* + * NOTE! We must preserve the order of this list! + */ enum { STORE_META_VOID, /* should not come up */ STORE_META_KEY_URL, /* key w/ keytype */ @@ -457,7 +460,8 @@ enum { STORE_META_URL, /* the url , if not in the header */ STORE_META_STD, /* standard metadata */ STORE_META_HITMETERING, /* reserved for hit metering */ - STORE_META_VALID + STORE_META_VALID, + STORE_META_END }; enum { diff --git a/src/globals.h b/src/globals.h index 7b2df947fb..cede3cd5db 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.29 1998/02/03 01:17:04 wessels Exp $ + * $Id: globals.h,v 1.30 1998/02/03 07:35:26 wessels Exp $ */ extern FILE *debug_log; /* NULL */ @@ -100,6 +100,7 @@ extern struct radix_node_head *AS_tree_head; extern double request_failure_ratio; /* 0.0 */ extern int store_hash_buckets; /* 0 */ extern hash_table *store_table; /* NULL */ +extern size_t cacheKeySize; #ifdef HAVE_SYSLOG extern int _db_level; diff --git a/src/protos.h b/src/protos.h index d8fdc600e0..891171697d 100644 --- a/src/protos.h +++ b/src/protos.h @@ -499,11 +499,10 @@ extern int storeVerifyCacheDirs(void); /* * store_swapmeta.c */ -extern int storeBuildMetaData(StoreEntry * e, char *swap_buf_c); -extern int getSwapHdr(int *, int *len, void *dst, char *write_buf, int hdr_len); -extern int getSwapHdr(int *, int *len, void *dst, char *write_buf, int hdr_len); -extern void addSwapHdr(int, int len, void *src, char *write_buf, int *write_len); -extern int storeGetMetaBuf(const char *buf, MemObject * mem); +char *storeSwapMetaPack(tlv *tlv_list, int *length); +tlv *storeSwapMetaBuild(StoreEntry * e); +tlv *storeSwapMetaUnpack(const char *buf, int *hdrlen); +void storeSwapTLVFree(tlv *n); /* * store_rebuild.c diff --git a/src/store.cc b/src/store.cc index 55eee6af81..7db352ae1c 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.372 1998/02/03 03:08:51 wessels Exp $ + * $Id: store.cc,v 1.373 1998/02/03 07:35:28 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -232,7 +232,6 @@ destroy_MemObject(StoreEntry * e) storeUnregister(e, mem->clients->callback_data); #endif assert(mem->clients == NULL); - safe_free(mem->swapout.meta_buf); memFree(MEM_HTTP_REPLY, mem->reply); safe_free(mem->url); safe_free(mem->log_url); @@ -1452,4 +1451,3 @@ storeBufferFlush(StoreEntry * e) EBIT_CLR(e->flag, DELAY_SENDING); InvokeHandlers(e); } - diff --git a/src/store_client.cc b/src/store_client.cc index 46c4536bfa..9a7aac54dc 100644 --- a/src/store_client.cc +++ b/src/store_client.cc @@ -208,12 +208,15 @@ storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, voi sc->disk_op_in_progress = 0; assert(sc->callback != NULL); debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); +#if USE_SWAP_HEADERS + /* XXX: BROKEN */ if (sc->copy_offset == 0 && len > 0 && mem != NULL) { hdr_len = storeGetMetaBuf(buf, mem); memmove((char *) buf, (char *) (buf + hdr_len), len - hdr_len); len -= hdr_len; httpParseReplyHeaders(buf, mem->reply); } +#endif sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, len); } diff --git a/src/store_key_md5.cc b/src/store_key_md5.cc index 28a7739dde..aae4c2dee6 100644 --- a/src/store_key_md5.cc +++ b/src/store_key_md5.cc @@ -1,5 +1,10 @@ #include "squid.h" +/* + * Size of a cache_key in bytes + */ +size_t cacheKeySize = MD5_DIGEST_CHARS; + const char * storeKeyText(const unsigned char *key) { diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc index 72cdfaed3f..ef790eeb02 100644 --- a/src/store_rebuild.cc +++ b/src/store_rebuild.cc @@ -12,14 +12,14 @@ struct _rebuild_dir { struct storeRebuildState { struct _rebuild_dir *rebuild_dir; - int objcount; /* # objects successfully reloaded */ - int expcount; /* # objects expired */ - int linecount; /* # lines parsed from cache logfile */ - int clashcount; /* # swapfile clashes avoided */ - int cancelcount; /* # objects cancelled */ - int dupcount; /* # duplicates purged */ - int invalid; /* # bad lines */ - int badflags; /* # bad e->flags */ + int objcount; /* # objects successfully reloaded */ + int expcount; /* # objects expired */ + int linecount; /* # lines parsed from cache logfile */ + int clashcount; /* # swapfile clashes avoided */ + int cancelcount; /* # objects cancelled */ + int dupcount; /* # duplicates purged */ + int invalid; /* # bad lines */ + int badflags; /* # bad e->flags */ int need_to_validate; time_t start; time_t stop; @@ -37,30 +37,28 @@ typedef struct valid_ctrl_t { static void storeRebuiltFromDisk(struct storeRebuildState *data); void -storeDoRebuildFromSwapFiles(void *data) +storeRebuildRawFile(void *data) { struct storeRebuildState *RB = data; - LOCAL_ARRAY(char, hdr_buf, 2 * MAX_URL); - LOCAL_ARRAY(cache_key, keybuf, MAX_URL); + LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE); StoreEntry *e = NULL; StoreEntry tmpe; int sfileno = 0; int count; int size; - int x; struct _rebuild_dir *d = RB->rebuild_dir; struct stat fst; static int filecount; - int hdr_len = 0; - int myt, myl; - int fd = 0; - debug(20, 3) (" Starting StoreRebuildFromSwapFiles at speed %d\n", d->speed); + int hdr_len; + int fd = -1; + tlv *tlv_list; + tlv *t; + debug(20, 3) (" Starting storeRebuildRawFile at speed %d\n", d->speed); for (count = 0; count < d->speed; count++) { - if (fd) - file_close(fd); + assert(fd == -1); fd = storeGetNextFile(&sfileno, &size); if (fd == -2) { - debug(20, 1) ("StoreRebuildFromSwapFiles: done!\n"); + debug(20, 1) ("storeRebuildRawFile: done!\n"); store_rebuilding = 0; return; } else if (fd == 0) { @@ -68,55 +66,61 @@ storeDoRebuildFromSwapFiles(void *data) } assert(fd > 0); /* lets get file stats here */ - x = fstat(fd, &fst); - assert(x == 0); - if ((++filecount & 0x3FFF) == 0) - debug(20, 1) (" %7d objects read so far.\n", RB->linecount); - debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); - x = read(fd, hdr_buf, 4096); - if (x < STORE_META_TLD_SIZE) { - debug(20, 1) (" Error reading header %s, small file, removing (read %d) %s\n", - xstrerror(), x, hdr_buf); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); + if (fstat(fd, &fst) < 0) { + debug(20, 1) ("storeRebuildRawFile: fstat(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + fd = -1; continue; } - if (SwapMetaType(hdr_buf) != META_OK) { - debug(20, 1) (" Found an old-style object or an invalid one\n"); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); + if ((++filecount & 0x3FFF) == 0) + debug(20, 1) (" %7d files processed so far.\n", RB->linecount); + debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); + if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) { + debug(20, 1) ("storeRebuildRawFile: read(FD %d): %s\n", + fd, xstrerror()); + file_close(fd); + fd = -1; continue; } - xmemcpy(&hdr_len, SwapMetaSize(hdr_buf), sizeof(int)); - if (x < hdr_len) { - debug(20, 1) (" Error header size > x (%d)%d\n", hdr_len, x); + file_close(fd); + fd = -1; + hdr_len = 0; + tlv_list = storeSwapMetaUnpack(hdr_buf, &hdr_len); + if (tlv_list == NULL) { + debug(20, 1) ("storeRebuildRawFile: failed to get meta data\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } - debug(20, 3) (" header size %d\n", hdr_len); - /* get key */ - if (0 == getSwapHdr(&myt, &myl, keybuf, hdr_buf, hdr_len)) { - debug(20, 1) ("Error getting STORE_META_KEY %d\n", x); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; + storeKeyFree(tmpe.key); + memset(&tmpe, '\0', sizeof(StoreEntry)); + for (t = tlv_list; t; t = t->next) { + switch (t->type) { + case STORE_META_KEY: + tmpe.key = storeKeyDup(t->value); + break; + case STORE_META_STD: + xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE); + break; + default: + break; + } } - keybuf[myl] = '\0'; - debug(20, 3) (" hm, we have %s, %d, %d\n", keybuf, myt, myl); - if (keybuf[0] == '\0' || myt != STORE_META_KEY) { - debug(20, 1) ("storeDoRebuildFromSwapFiles: bad key\n"); + storeSwapTLVFree(tlv_list); + tlv_list = NULL; + if (tmpe.key == NULL) { + debug(20, 1) ("storeRebuildRawFile: NULL key\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } - /* get the standard meta data for the StoreEntry */ - memset(&tmpe, '\0', sizeof(StoreEntry)); - if (0 == getSwapHdr(&myt, &myl, &tmpe.timestamp, hdr_buf, hdr_len)) { - debug(20, 1) ("storeDoRebuildFromSwapFiles:Error getting STORE_META_STD %d\n", myl); + if (tmpe.object_len == 0) { + debug(20, 1) ("storeRebuildRawFile: ZERO object length\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } - assert(myt == STORE_META_STD); - assert(myl == STORE_HDR_METASIZE); /* check sizes */ if (hdr_len + tmpe.object_len != fst.st_size) { - debug(20, 1) ("storeDoRebuildFromSwapFiles:INVALID swapfile, sizes dont match %d+%d!=%d\n", + debug(20, 1) ("storeRebuildRawFile: SIZE MISMATCH %d+%d!=%d\n", hdr_len, tmpe.object_len, fst.st_size); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; @@ -126,7 +130,7 @@ storeDoRebuildFromSwapFiles(void *data) RB->badflags++; continue; } - if ((e = storeGet(keybuf)) != NULL) { + if ((e = storeGet(tmpe.key)) != NULL) { /* URL already exists, this swapfile not being used */ /* junk old, load new */ storeRelease(e); /* release old entry */ @@ -134,24 +138,19 @@ storeDoRebuildFromSwapFiles(void *data) } /* update store_swap_size */ RB->objcount++; - debug(20, 4) ("storeDoRebuildFromSwapFiles: KEY=%20s , sfileno=%08X exp=%08X timest=%08X\n", - keybuf, sfileno, tmpe.expires, tmpe.timestamp); - debug(20, 4) (" lastref=%08X lastmod=%08X refcount=%08X flag=%08X\n", - tmpe.lastref, tmpe.lastmod, tmpe.refcount, tmpe.flag); - debug(20, 4) (" len=%d hdr_len=%d file_len=%d\n", tmpe.object_len, - hdr_len, fst.st_size); - e = storeAddDiskRestore(keybuf, + storeEntryDump(&tmpe); + e = storeAddDiskRestore(tmpe.key, sfileno, (int) tmpe.object_len, tmpe.expires, tmpe.timestamp, tmpe.lastref, tmpe.lastmod, - (u_num32) tmpe.refcount, /* refcount */ - (u_num32) tmpe.flag, /* flags */ + tmpe.refcount, /* refcount */ + tmpe.flag, /* flags */ d->clean); } - eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); + eventAdd("storeRebuild", storeRebuildRawFile, RB, 0); } @@ -201,7 +200,9 @@ storeConvertFile(const cache_key * key, int fd_r, fd_w; int hdr_len, x, y; LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, copybuf, STORE_META_BUFSZ); + LOCAL_ARRAY(char, copybuf, DISK_PAGE_SIZE); + char *buf; + tlv *tlv_list; StoreEntry e; e.key = key; e.object_len = size; @@ -210,21 +211,22 @@ storeConvertFile(const cache_key * key, e.refcount = refcount; e.flag = flags; storeSwapFullPath(file_number, swapfilename); - fd_r = open(swapfilename, O_RDONLY); - if (fd_r < 0) { /* ERROR */ + fd_r = file_open(swapfilename, O_RDONLY, NULL, NULL, NULL); + if (fd_r < 0) return; - } safeunlink(swapfilename, 1); - fd_w = open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC); - hdr_len = storeBuildMetaData(&e, copybuf); - assert(hdr_len < STORE_META_BUFSZ); - x = write(fd_w, copybuf, hdr_len); + fd_w = file_open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC, NULL, NULL, NULL); + tlv_list = storeSwapMetaBuild(&e); + buf = storeSwapMetaPack(tlv_list, &hdr_len); + x = write(fd_w, buf, hdr_len); while (x > 0) { - y = read(fd_r, copybuf, STORE_META_BUFSZ); + y = read(fd_r, copybuf, DISK_PAGE_SIZE); x = write(fd_w, copybuf, y); } - close(fd_r); - close(fd_w); + file_close(fd_r); + file_close(fd_w); + xfree(buf); + storeSwapTLVFree(tlv_list); } int @@ -515,9 +517,7 @@ storeDoConvertFromLog(void *data) (u_short) scan6, /* refcount */ (u_short) scan7, /* flags */ d->clean); -#if 0 storeDirSwapLog(e); -#endif } RB->rebuild_dir = d->next; for (D = &RB->rebuild_dir; *D; D = &(*D)->next); @@ -678,8 +678,8 @@ storeStartRebuildFromDisk(void) debug(20, 1) ("Rebuilding storage (%s)\n", clean ? "CLEAN" : "DIRTY"); if (opt_foreground_rebuild) { - storeDoRebuildFromSwapFiles(RB); + storeRebuildRawFile(RB); } else { - eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); + eventAdd("storeRebuild", storeRebuildRawFile, RB, 0); } } diff --git a/src/store_swapin.cc b/src/store_swapin.cc index c2716e079b..649cf2ae4c 100644 --- a/src/store_swapin.cc +++ b/src/store_swapin.cc @@ -39,13 +39,11 @@ storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) storeValidate(e, storeSwapInValidateComplete, ctrlp, callback_data); } - void storeSwapInValidateComplete(void *data, int retcode, int errcode) { swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; StoreEntry *e; - if (retcode == -2 && errcode == -2) { xfree(ctrlp); return; @@ -70,7 +68,6 @@ storeSwapInFileOpened(void *data, int fd, int errcode) StoreEntry *e = ctrlp->e; MemObject *mem = e->mem_obj; struct stat sb; - if (fd == -2 && errcode == -2) { xfree(ctrlp->path); xfree(ctrlp); @@ -100,3 +97,4 @@ storeSwapInFileOpened(void *data, int fd, int errcode) xfree(ctrlp->path); xfree(ctrlp); } + diff --git a/src/store_swapmeta.cc b/src/store_swapmeta.cc index 8d30e649f6..269fc3aaa6 100644 --- a/src/store_swapmeta.cc +++ b/src/store_swapmeta.cc @@ -1,114 +1,109 @@ #include "squid.h" -#define squid_key_size MD5_DIGEST_CHARS - -/* build swapfile header */ -int -storeBuildMetaData(StoreEntry * e, char *swap_buf_c) +static tlv ** +storeSwapTLVAdd(int type, const void *ptr, size_t len, tlv ** tail) { - MemObject *mem; - int a = STORE_META_TLD_START; - char *meta_buf; - mem = e->mem_obj; - meta_buf = mem->swapout.meta_buf; - debug(20, 3) ("storeBuildSwapFileHeader: called.\n"); - assert(e->swap_status == SWAPOUT_WRITING); - if (!meta_buf) - meta_buf = mem->swapout.meta_buf = xmalloc(1024); - /* construct header */ - /* add Length(int)-Type(char)-Data encoded info */ - meta_buf[0] = META_OK; - xmemcpy(&meta_buf[1], &a, sizeof(int)); - mem->swapout.meta_len = STORE_META_TLD_START; - addSwapHdr(STORE_META_KEY, squid_key_size, (void *) e->key, - mem->swapout.meta_buf, &mem->swapout.meta_len); - addSwapHdr(STORE_META_STD, STORE_HDR_METASIZE, (void *) &e->timestamp, - mem->swapout.meta_buf, &mem->swapout.meta_len); - debug(20, 3) ("storeBuildSwapFileHeader: len=%d.\n", mem->swapout.meta_len); - if (swap_buf_c) - xmemcpy(swap_buf_c, mem->swapout.meta_buf, mem->swapout.meta_len); - return mem->swapout.meta_len; + tlv *t = xcalloc(1, sizeof(tlv)); + t->type = (char) type; + t->length = (int) len; + t->value = xmalloc(len); + xmemcpy(t->value, ptr, len); + *tail = t; + return &t->next; /* return new tail pointer */ } -int -getSwapHdr(int *type, int *len, void *dst, char *write_buf, int hdr_len) +void +storeSwapTLVFree(tlv * n) { - static int cur = 0; - static char *curptr; - char *tmp_buf; - if (cur == 0 || curptr != write_buf) { /* first call or rewind ! */ - cur = STORE_META_TLD_START; - curptr = write_buf; + tlv *t; + while ((t = n) != NULL) { + n = t->next; + xfree(t->value); + xfree(t); } - if (cur + STORE_META_TLD_START > hdr_len) { - debug(20, 3) ("getSwapHdr: overflow, %d %d.\n", cur, hdr_len); - cur = 0; - return -1; - } - tmp_buf = &write_buf[cur]; /* position ourselves */ - xmemcpy(len, SwapMetaSize(tmp_buf), sizeof(int)); /* length */ - *type = SwapMetaType(tmp_buf); /* type */ - xmemcpy(dst, SwapMetaData(tmp_buf), *len); /* data */ - cur += STORE_META_TLD_START + *len; /* advance position */ - debug(20, 4) ("getSwapHdr: t=%d l=%d (cur=%d hdr_len=%d) (%p)\n", - *type, *len, cur, hdr_len, dst); - if (cur == hdr_len) { - debug(20, 4) ("getSwapHdr: finished with this.\n"); - cur = 0; - return 1; - } - return 1; /* ok ! */ } -void -addSwapHdr(int type, int len, void *src, char *write_buf, int *write_len) +/* + * Build a TLV list for a StoreEntry + */ +tlv * +storeSwapMetaBuild(StoreEntry * e) { - int hdr_len = *write_len; - char *base = &write_buf[hdr_len]; - debug(20, 3) ("addSwapHdr: at %d\n", hdr_len); - base[0] = (char) type; - xmemcpy(&base[1], &len, sizeof(int)); - xmemcpy(SwapMetaData(base), src, len); - hdr_len += STORE_META_TLD_START + len; - /* now we know length */ - debug(20, 3) ("addSwapHdr: added type=%d len=%d data=%p. hdr_len=%d\n", - type, len, src, hdr_len); - /* update header */ - xmemcpy(&write_buf[1], &hdr_len, sizeof(int)); - *write_len = hdr_len; + MemObject *mem = e->mem_obj; + tlv *TLV = NULL; /* we'll return this */ + tlv **T = &TLV; + const char *url; + assert(mem != NULL); + assert(e->swap_status == SWAPOUT_WRITING); + url = storeUrl(e); + debug(20, 3) ("storeSwapMetaBuild: %s\n", url); + T = storeSwapTLVAdd(STORE_META_KEY, e->key, cacheKeySize, T); + T = storeSwapTLVAdd(STORE_META_STD, &e->timestamp, STORE_HDR_METASIZE, T); + T = storeSwapTLVAdd(STORE_META_URL, url, strlen(url), T); + return TLV; } -int -storeGetMetaBuf(const char *buf, MemObject * mem) +char * +storeSwapMetaPack(tlv * tlv_list, int *length) { - int hdr_len; - assert(mem != NULL); - /* the key */ - if (SwapMetaType(buf) != META_OK) { - debug(20, 1) ("storeGetMetaBuf:Found an old-style object, damn.\n"); - return -1; + size_t buflen = 0; + int i; + tlv *t; + off_t j = 0; + char *buf; + assert(length != NULL); + buflen++; /* STORE_META_OK */ + buflen += sizeof(int); /* size of header to follow */ + for (t = tlv_list; t; t = t->next) + buflen += t->length + sizeof(char) + sizeof(int); + buflen++; /* STORE_META_END */ + buf = xmalloc(buflen); + buf[j++] = (char) STORE_META_OK; + i = (int) buflen - (sizeof(char) + sizeof(int)); + xmemcpy(&buf[j], &i, sizeof(int)); + for (t = tlv_list; t; t = t->next) { + buf[j++] = (char) t->type; + xmemcpy(&buf[j], &t->length, sizeof(int)); + j += sizeof(int); + xmemcpy(&buf[j], t->value, t->length); + j += t->length; } - xmemcpy(&hdr_len, SwapMetaSize(buf), sizeof(int)); - mem->swapout.meta_len = hdr_len; - mem->swapout.meta_buf = xmalloc(hdr_len); - xmemcpy(mem->swapout.meta_buf, buf, hdr_len); - debug(20, 3) (" header size %d\n", hdr_len); - return hdr_len; + buf[j++] = (char) STORE_META_END; + assert(j == buflen); + *length = (int) buflen; + return buf; } -#if OLD_CODE -static int -storeParseMetaBuf(StoreEntry * e) +tlv * +storeSwapMetaUnpack(const char *buf, int *hdr_len) { - static char mbuf[1024]; - int myt, myl; - MemObject *mem = e->mem_obj; - assert(e && e->mem_obj && e->key); - getSwapHdr(&myt, &myl, mbuf, mem->swapout.meta_buf, mem->swapout.meta_len); - mbuf[myl] = 0; - debug(20, 3) ("storeParseMetaBuf: key=%s\n", mbuf); - e->key = xstrdup(storeKeyScan(mbuf)); - getSwapHdr(&myt, &myl, &e->timestamp, mem->swapout.meta_buf, mem->swapout.meta_len); - return 1; + tlv *TLV; /* we'll return this */ + tlv **T = &TLV; + char type; + int length; + int buflen; + off_t j = 0; + assert(buflen > (sizeof(char) + sizeof(int))); + assert(buf != NULL); + assert(hdr_len != NULL); + if (buf[j++] != (char) STORE_META_OK) + return NULL; + xmemcpy(&buflen, &buf[j], sizeof(int)); + j += sizeof(int); + assert(buflen > 0); + while (buflen - j > (sizeof(char) + sizeof(int))) { + type = buf[j++]; + xmemcpy(&length, &buf[j], sizeof(int)); + j += sizeof(int); + if (j + length > buflen) { + debug(20, 0) ("storeSwapMetaUnpack: overflow!\n"); + debug(20, 0) ("\ttype=%d, length=%d, buflen=%d, offset=%d\n", + type, length, buflen, (int) j); + break; + } + T = storeSwapTLVAdd(type, &buf[j], (size_t) length, T); + j += length; + } + *hdr_len = buflen + sizeof(char) + sizeof(int); + return TLV; } -#endif diff --git a/src/store_swapout.cc b/src/store_swapout.cc index 79736d40ba..f4a9b5f565 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -71,7 +71,10 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) assert(mem != NULL); #endif mem->swapout.done_offset += len; - if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len + mem->swapout.meta_len) { + if (e->store_status == STORE_PENDING) { + storeCheckSwapOut(e); + return; + } else if (mem->swapout.done_offset < e->object_len + mem->swapout.hdr_len) { storeCheckSwapOut(e); return; } @@ -82,9 +85,7 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); -#if 0 storeDirSwapLog(e); -#endif } /* Note, we don't otherwise call storeReleaseRequest() here because * storeCheckCachable() does it for is if necessary */ @@ -169,16 +170,27 @@ storeCheckSwapOut(StoreEntry * e) return; assert(mem->swapout.fd > -1); swap_buf = memAllocate(MEM_DISK_BUF, 1); - if (mem->swapout.queue_offset == 0) - hdr_len = storeBuildMetaData(e, swap_buf); +#if USE_SWAP_HEADER + /* XXX: BROKEN */ + if (mem->swapout.queue_offset == 0) { + tlv = storeSwapMetaBuild(e); + hdr_buf = storeSwapMetaPack(tlv, &hdr_len); + } if (swapout_size > STORE_SWAP_BUF - hdr_len) swapout_size = STORE_SWAP_BUF - hdr_len; - swap_buf_len = stmemCopy(mem->data, mem->swapout.queue_offset, swap_buf + hdr_len, swapout_size) + hdr_len; +#else + if (swapout_size > STORE_SWAP_BUF) + swapout_size = STORE_SWAP_BUF; + swap_buf_len = stmemCopy(mem->data, + mem->swapout.queue_offset, + swap_buf, + swapout_size); +#endif if (swap_buf_len < 0) { debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); diff --git a/src/structs.h b/src/structs.h index ad21d2a765..20686c8155 100644 --- a/src/structs.h +++ b/src/structs.h @@ -1,8 +1,5 @@ - - - struct _acl_ip_data { struct in_addr addr1; /* if addr2 non-zero then its a range */ struct in_addr addr2; @@ -807,8 +804,7 @@ struct _MemObject { off_t queue_offset; off_t done_offset; int fd; - int meta_len; - char *meta_buf; + size_t hdr_len; /* size of the swapfile header */ } swapout; struct _http_reply *reply; request_t *request; @@ -953,3 +949,10 @@ struct _StatCounters { double cputime; struct timeval timestamp; }; + +struct _tlv { + char type; + int length; + void *value; + struct _tlv *next; +}; diff --git a/src/typedefs.h b/src/typedefs.h index 806a16738b..48662c0bc3 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -76,6 +76,7 @@ typedef struct _ErrorState ErrorState; typedef struct _dlink_node dlink_node; typedef struct _dlink_list dlink_list; typedef struct _StatCounters StatCounters; +typedef struct _tlv tlv; /* define AIOCB even without USE_ASYNC_IO */ typedef void AIOCB(void *, int aio_return, int aio_errno); -- 2.47.2