#
# Makefile for the Squid Object Cache server
#
-# $Id: Makefile.am,v 1.34 2002/10/02 11:06:31 robertc Exp $
+# $Id: Makefile.am,v 1.35 2002/10/12 09:45:55 robertc Exp $
#
# Uncomment and customize the following to suit your needs:
#
structs.h \
tools.c \
typedefs.h \
+ ufscommon.c \
+ ufscommon.h \
$(UNLINKDSOURCE) \
url.c \
urn.c \
#
# Makefile for the Squid Object Cache server
#
-# $Id: Makefile.in,v 1.248 2002/10/02 11:06:31 robertc Exp $
+# $Id: Makefile.in,v 1.249 2002/10/12 09:45:55 robertc Exp $
#
# Uncomment and customize the following to suit your needs:
#
structs.h \
tools.c \
typedefs.h \
+ ufscommon.c \
+ ufscommon.h \
$(UNLINKDSOURCE) \
url.c \
urn.c \
pinger_LDADD = $(LDADD)
pinger_DEPENDENCIES =
pinger_LDFLAGS =
-@USE_DELAY_POOLS_FALSE@am__objects_4 =
-@USE_DELAY_POOLS_TRUE@am__objects_4 = delay_pools.$(OBJEXT)
-@USE_DNSSERVER_FALSE@am__objects_5 = dns_internal.$(OBJEXT)
-@USE_DNSSERVER_TRUE@am__objects_5 = dns.$(OBJEXT)
-@ENABLE_HTCP_TRUE@am__objects_6 = htcp.$(OBJEXT)
-@MAKE_LEAKFINDER_TRUE@am__objects_7 = leakfinder.$(OBJEXT)
-@MAKE_LEAKFINDER_FALSE@am__objects_7 =
-@ENABLE_XPROF_STATS_TRUE@am__objects_3 = ProfStats.$(OBJEXT)
-@ENABLE_XPROF_STATS_FALSE@am__objects_3 =
-@USE_SNMP_TRUE@am__objects_8 = snmp_core.$(OBJEXT) snmp_agent.$(OBJEXT)
-@USE_SNMP_FALSE@am__objects_8 =
-@ENABLE_SSL_FALSE@am__objects_9 =
-@ENABLE_SSL_TRUE@am__objects_9 = ssl_support.$(OBJEXT)
-@ENABLE_UNLINKD_TRUE@am__objects_10 = unlinkd.$(OBJEXT)
-@ENABLE_UNLINKD_FALSE@am__objects_10 =
-@ENABLE_WIN32SPECIFIC_FALSE@am__objects_11 =
-@ENABLE_WIN32SPECIFIC_TRUE@am__objects_11 = win32.$(OBJEXT)
+@USE_DELAY_POOLS_FALSE@am__objects_1 =
+@USE_DELAY_POOLS_TRUE@am__objects_1 = delay_pools.$(OBJEXT)
+@USE_DNSSERVER_FALSE@am__objects_2 = dns_internal.$(OBJEXT)
+@USE_DNSSERVER_TRUE@am__objects_2 = dns.$(OBJEXT)
+@ENABLE_HTCP_TRUE@am__objects_3 = htcp.$(OBJEXT)
+@MAKE_LEAKFINDER_TRUE@am__objects_4 = leakfinder.$(OBJEXT)
+@MAKE_LEAKFINDER_FALSE@am__objects_4 =
+@ENABLE_XPROF_STATS_TRUE@am__objects_5 = ProfStats.$(OBJEXT)
+@ENABLE_XPROF_STATS_FALSE@am__objects_5 =
+@USE_SNMP_TRUE@am__objects_6 = snmp_core.$(OBJEXT) snmp_agent.$(OBJEXT)
+@USE_SNMP_FALSE@am__objects_6 =
+@ENABLE_SSL_FALSE@am__objects_7 =
+@ENABLE_SSL_TRUE@am__objects_7 = ssl_support.$(OBJEXT)
+@ENABLE_UNLINKD_TRUE@am__objects_8 = unlinkd.$(OBJEXT)
+@ENABLE_UNLINKD_FALSE@am__objects_8 =
+@ENABLE_WIN32SPECIFIC_FALSE@am__objects_9 =
+@ENABLE_WIN32SPECIFIC_TRUE@am__objects_9 = win32.$(OBJEXT)
am_squid_OBJECTS = access_log.$(OBJEXT) acl.$(OBJEXT) asn.$(OBJEXT) \
authenticate.$(OBJEXT) cache_cf.$(OBJEXT) CacheDigest.$(OBJEXT) \
cache_manager.$(OBJEXT) carp.$(OBJEXT) cbdata.$(OBJEXT) \
client_side_reply.$(OBJEXT) client_side_request.$(OBJEXT) \
clientStream.$(OBJEXT) comm.$(OBJEXT) comm_select.$(OBJEXT) \
comm_poll.$(OBJEXT) comm_kqueue.$(OBJEXT) debug.$(OBJEXT) \
- $(am__objects_4) disk.$(OBJEXT) $(am__objects_5) \
+ $(am__objects_1) disk.$(OBJEXT) $(am__objects_2) \
errorpage.$(OBJEXT) ETag.$(OBJEXT) event.$(OBJEXT) \
external_acl.$(OBJEXT) fd.$(OBJEXT) filemap.$(OBJEXT) \
forward.$(OBJEXT) fqdncache.$(OBJEXT) ftp.$(OBJEXT) \
- gopher.$(OBJEXT) helper.$(OBJEXT) $(am__objects_6) \
+ gopher.$(OBJEXT) helper.$(OBJEXT) $(am__objects_3) \
http.$(OBJEXT) HttpStatusLine.$(OBJEXT) HttpHdrCc.$(OBJEXT) \
HttpHdrRange.$(OBJEXT) HttpHdrContRange.$(OBJEXT) \
HttpHeader.$(OBJEXT) HttpHeaderTools.$(OBJEXT) \
HttpRequest.$(OBJEXT) icmp.$(OBJEXT) icp_v2.$(OBJEXT) \
icp_v3.$(OBJEXT) ident.$(OBJEXT) internal.$(OBJEXT) \
ipc.$(OBJEXT) ipcache.$(OBJEXT) IPInterception.$(OBJEXT) \
- $(am__objects_7) logfile.$(OBJEXT) main.$(OBJEXT) mem.$(OBJEXT) \
+ $(am__objects_4) logfile.$(OBJEXT) main.$(OBJEXT) mem.$(OBJEXT) \
MemBuf.$(OBJEXT) mime.$(OBJEXT) multicast.$(OBJEXT) \
neighbors.$(OBJEXT) net_db.$(OBJEXT) Packer.$(OBJEXT) \
- $(am__objects_3) pconn.$(OBJEXT) peer_digest.$(OBJEXT) \
+ $(am__objects_5) pconn.$(OBJEXT) peer_digest.$(OBJEXT) \
peer_select.$(OBJEXT) redirect.$(OBJEXT) referer.$(OBJEXT) \
- refresh.$(OBJEXT) send-announce.$(OBJEXT) $(am__objects_8) \
- ssl.$(OBJEXT) $(am__objects_9) stat.$(OBJEXT) \
+ refresh.$(OBJEXT) send-announce.$(OBJEXT) $(am__objects_6) \
+ ssl.$(OBJEXT) $(am__objects_7) stat.$(OBJEXT) \
StatHist.$(OBJEXT) String.$(OBJEXT) stmem.$(OBJEXT) \
store.$(OBJEXT) store_io.$(OBJEXT) store_client.$(OBJEXT) \
store_digest.$(OBJEXT) store_dir.$(OBJEXT) \
store_key_md5.$(OBJEXT) store_log.$(OBJEXT) \
store_rebuild.$(OBJEXT) store_swapin.$(OBJEXT) \
store_swapmeta.$(OBJEXT) store_swapout.$(OBJEXT) \
- tools.$(OBJEXT) $(am__objects_10) url.$(OBJEXT) urn.$(OBJEXT) \
- useragent.$(OBJEXT) wais.$(OBJEXT) wccp.$(OBJEXT) \
- whois.$(OBJEXT) $(am__objects_11)
+ tools.$(OBJEXT) ufscommon.$(OBJEXT) $(am__objects_8) \
+ url.$(OBJEXT) urn.$(OBJEXT) useragent.$(OBJEXT) wais.$(OBJEXT) \
+ wccp.$(OBJEXT) whois.$(OBJEXT) $(am__objects_9)
nodist_squid_OBJECTS = repl_modules.$(OBJEXT) auth_modules.$(OBJEXT) \
store_modules.$(OBJEXT) globals.$(OBJEXT) \
string_arrays.$(OBJEXT)
@AMDEP_TRUE@ $(DEPDIR)/store_swapmeta.Po \
@AMDEP_TRUE@ $(DEPDIR)/store_swapout.Po \
@AMDEP_TRUE@ $(DEPDIR)/string_arrays.Po $(DEPDIR)/tools.Po \
-@AMDEP_TRUE@ $(DEPDIR)/unlinkd.Po $(DEPDIR)/url.Po \
-@AMDEP_TRUE@ $(DEPDIR)/urn.Po $(DEPDIR)/useragent.Po \
-@AMDEP_TRUE@ $(DEPDIR)/wais.Po $(DEPDIR)/wccp.Po \
-@AMDEP_TRUE@ $(DEPDIR)/whois.Po $(DEPDIR)/win32.Po
+@AMDEP_TRUE@ $(DEPDIR)/ufscommon.Po $(DEPDIR)/unlinkd.Po \
+@AMDEP_TRUE@ $(DEPDIR)/url.Po $(DEPDIR)/urn.Po \
+@AMDEP_TRUE@ $(DEPDIR)/useragent.Po $(DEPDIR)/wais.Po \
+@AMDEP_TRUE@ $(DEPDIR)/wccp.Po $(DEPDIR)/whois.Po \
+@AMDEP_TRUE@ $(DEPDIR)/win32.Po
COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
CCLD = $(CC)
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/store_swapout.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/string_arrays.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/tools.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/ufscommon.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/unlinkd.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/url.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/urn.Po@am__quote@
void aioSync(SwapDir *);
int aioQueueSize(void);
-struct _squidaioinfo_t {
- int swaplog_fd;
- int l1;
- int l2;
- fileMap *map;
- int suggest;
-};
-
struct _squidaiostate_t {
int fd;
struct {
void *callback_data;
};
-typedef struct _squidaioinfo_t squidaioinfo_t;
typedef struct _squidaiostate_t squidaiostate_t;
/* The squidaio_state memory pools */
extern MemPool *aufs_qread_pool;
extern MemPool *aufs_qwrite_pool;
-extern void storeAufsDirMapBitReset(SwapDir *, sfileno);
-extern int storeAufsDirMapBitAllocate(SwapDir *);
-
-extern char *storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath);
-extern void storeAufsDirUnlinkFile(SwapDir *, sfileno);
-extern void storeAufsDirReplAdd(SwapDir * SD, StoreEntry *);
-extern void storeAufsDirReplRemove(StoreEntry *);
-
/*
* Store IO stuff
*/
/*
- * $Id: store_dir_aufs.cc,v 1.49 2002/08/23 22:31:09 hno Exp $
+ * $Id: store_dir_aufs.cc,v 1.50 2002/10/12 09:45:56 robertc Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
#include "squid.h"
#include "store_asyncufs.h"
+#include "ufscommon.h"
-#define DefaultLevelOneDirs 16
-#define DefaultLevelTwoDirs 256
-#define STORE_META_BUFSZ 4096
-
-typedef struct _RebuildState RebuildState;
-struct _RebuildState {
- SwapDir *sd;
- int n_read;
- FILE *log;
- int speed;
- int curlvl1;
- int curlvl2;
- struct {
- unsigned int need_to_validate:1;
- unsigned int clean:1;
- unsigned int init:1;
- } flags;
- int done;
- int in_dir;
- int fn;
- struct dirent *entry;
- DIR *td;
- char fullpath[SQUID_MAXPATHLEN];
- char fullfilename[SQUID_MAXPATHLEN];
- struct _store_rebuild_data counts;
-};
-
-static int n_asyncufs_dirs = 0;
-static int *asyncufs_dir_index = NULL;
MemPool *squidaio_state_pool = NULL;
MemPool *aufs_qread_pool = NULL;
MemPool *aufs_qwrite_pool = NULL;
static int asyncufs_initialised = 0;
-static char *storeAufsDirSwapSubDir(SwapDir *, int subdirn);
-static int storeAufsDirCreateDirectory(const char *path, int);
-static int storeAufsDirVerifyCacheDirs(SwapDir *);
-static int storeAufsDirVerifyDirectory(const char *path);
-static void storeAufsDirCreateSwapSubDirs(SwapDir *);
-static char *storeAufsDirSwapLogFile(SwapDir *, const char *);
-static EVH storeAufsDirRebuildFromDirectory;
-static EVH storeAufsDirRebuildFromSwapLog;
-static int storeAufsDirGetNextFile(RebuildState *, sfileno *, int *size);
-static StoreEntry *storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- sfileno file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean);
-static void storeAufsDirRebuild(SwapDir * sd);
-static void storeAufsDirCloseTmpSwapLog(SwapDir * sd);
-static FILE *storeAufsDirOpenTmpSwapLog(SwapDir *, int *, int *);
-static STLOGOPEN storeAufsDirOpenSwapLog;
-static STINIT storeAufsDirInit;
-static STFREE storeAufsDirFree;
-static STLOGCLEANSTART storeAufsDirWriteCleanStart;
-static STLOGCLEANNEXTENTRY storeAufsDirCleanLogNextEntry;
-static STLOGCLEANWRITE storeAufsDirWriteCleanEntry;
-static STLOGCLEANDONE storeAufsDirWriteCleanDone;
-static STLOGCLOSE storeAufsDirCloseSwapLog;
-static STLOGWRITE storeAufsDirSwapLog;
-static STNEWFS storeAufsDirNewfs;
static STDUMP storeAufsDirDump;
-static STMAINTAINFS storeAufsDirMaintain;
static STCHECKOBJ storeAufsDirCheckObj;
-static STREFOBJ storeAufsDirRefObj;
-static STUNREFOBJ storeAufsDirUnrefObj;
-static QS rev_int_sort;
-static int storeAufsDirClean(int swap_index);
-static EVH storeAufsDirCleanEvent;
-static int storeAufsDirIs(SwapDir * sd);
-static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2);
-static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *);
-static void storeAufsDirStats(SwapDir *, StoreEntry *);
-static void storeAufsDirInitBitmap(SwapDir *);
-static int storeAufsDirValidFileno(SwapDir *, sfileno, int);
+static void storeAufsDirIOUnlinkFile(char *path);
+
/* The MAIN externally visible function */
STSETUP storeFsSetup_aufs;
-/*
- * These functions were ripped straight out of the heart of store_dir.c.
- * They assume that the given filenum is on a asyncufs partiton, which may or
- * may not be true..
- * XXX this evilness should be tidied up at a later date!
- */
-
-static int
-storeAufsDirMapBitTest(SwapDir * SD, sfileno filn)
-{
- squidaioinfo_t *aioinfo;
- aioinfo = (squidaioinfo_t *) SD->fsdata;
- return file_map_bit_test(aioinfo->map, filn);
-}
-
-static void
-storeAufsDirMapBitSet(SwapDir * SD, sfileno filn)
-{
- squidaioinfo_t *aioinfo;
- aioinfo = (squidaioinfo_t *) SD->fsdata;
- file_map_bit_set(aioinfo->map, filn);
-}
-
-void
-storeAufsDirMapBitReset(SwapDir * SD, sfileno filn)
-{
- squidaioinfo_t *aioinfo;
- aioinfo = (squidaioinfo_t *) SD->fsdata;
- /*
- * We have to test the bit before calling file_map_bit_reset.
- * file_map_bit_reset doesn't do bounds checking. It assumes
- * filn is a valid file number, but it might not be because
- * the map is dynamic in size. Also clearing an already clear
- * bit puts the map counter of-of-whack.
- */
- if (file_map_bit_test(aioinfo->map, filn))
- file_map_bit_reset(aioinfo->map, filn);
-}
-
-int
-storeAufsDirMapBitAllocate(SwapDir * SD)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata;
- int fn;
- fn = file_map_allocate(aioinfo->map, aioinfo->suggest);
- file_map_bit_set(aioinfo->map, fn);
- aioinfo->suggest = fn + 1;
- return fn;
-}
-
-/*
- * Initialise the asyncufs bitmap
- *
- * If there already is a bitmap, and the numobjects is larger than currently
- * configured, we allocate a new bitmap and 'grow' the old one into it.
- */
-static void
-storeAufsDirInitBitmap(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
-
- if (aioinfo->map == NULL) {
- /* First time */
- aioinfo->map = file_map_create();
- } else if (aioinfo->map->max_n_files) {
- /* it grew, need to expand */
- /* XXX We don't need it anymore .. */
- }
- /* else it shrunk, and we leave the old one in place */
-}
-
-static char *
-storeAufsDirSwapSubDir(SwapDir * sd, int subdirn)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
-
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- assert(0 <= subdirn && subdirn < aioinfo->l1);
- snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
- return fullfilename;
-}
-
-static int
-storeAufsDirCreateDirectory(const char *path, int should_exist)
-{
- int created = 0;
- struct stat st;
- getCurrentTime();
- if (0 == stat(path, &st)) {
- if (S_ISDIR(st.st_mode)) {
- debug(47, should_exist ? 3 : 1) ("%s exists\n", path);
- } else {
- fatalf("Swap directory %s is not a directory.", path);
- }
- } else if (0 == mkdir(path, 0755)) {
- debug(47, should_exist ? 1 : 3) ("%s created\n", path);
- created = 1;
- } else {
- fatalf("Failed to make swap directory %s: %s",
- path, xstrerror());
- }
- return created;
-}
-
-static int
-storeAufsDirVerifyDirectory(const char *path)
-{
- struct stat sb;
- if (stat(path, &sb) < 0) {
- debug(47, 0) ("%s: %s\n", path, xstrerror());
- return -1;
- }
- if (S_ISDIR(sb.st_mode) == 0) {
- debug(47, 0) ("%s is not a directory\n", path);
- return -1;
- }
- return 0;
-}
-
-/*
- * This function is called by storeAufsDirInit(). If this returns < 0,
- * then Squid exits, complains about swap directories not
- * existing, and instructs the admin to run 'squid -z'
- */
-static int
-storeAufsDirVerifyCacheDirs(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- int j;
- const char *path = sd->path;
-
- if (storeAufsDirVerifyDirectory(path) < 0)
- return -1;
- for (j = 0; j < aioinfo->l1; j++) {
- path = storeAufsDirSwapSubDir(sd, j);
- if (storeAufsDirVerifyDirectory(path) < 0)
- return -1;
- }
- return 0;
-}
-
-static void
-storeAufsDirCreateSwapSubDirs(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- int i, k;
- int should_exist;
- LOCAL_ARRAY(char, name, MAXPATHLEN);
- for (i = 0; i < aioinfo->l1; i++) {
- snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
- if (storeAufsDirCreateDirectory(name, 0))
- should_exist = 0;
- else
- should_exist = 1;
- debug(47, 1) ("Making directories in %s\n", name);
- for (k = 0; k < aioinfo->l2; k++) {
- snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
- storeAufsDirCreateDirectory(name, should_exist);
- }
- }
-}
-
-static char *
-storeAufsDirSwapLogFile(SwapDir * sd, const char *ext)
-{
- LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, digit, 32);
- char *pathtmp2;
- if (Config.Log.swap) {
- xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
- pathtmp2 = pathtmp;
- while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
- *pathtmp2 = '.';
- while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
- pathtmp[strlen(pathtmp) - 1] = '\0';
- for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
- snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
- if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
- strcat(path, ".");
- snprintf(digit, 32, "%02d", sd->index);
- strncat(path, digit, 3);
- }
- } else {
- xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
- strcat(path, "/swap.state");
- }
- if (ext)
- strncat(path, ext, 16);
- return path;
-}
-
-static void
-storeAufsDirOpenSwapLog(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- char *path;
- int fd;
- path = storeAufsDirSwapLogFile(sd, NULL);
- fd = file_open(path, O_WRONLY | O_CREAT | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", path, xstrerror());
- fatal("storeAufsDirOpenSwapLog: Failed to open swap log.");
- }
- debug(50, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
- aioinfo->swaplog_fd = fd;
- if (0 == n_asyncufs_dirs)
- assert(NULL == asyncufs_dir_index);
- n_asyncufs_dirs++;
- assert(n_asyncufs_dirs <= Config.cacheSwap.n_configured);
-}
-
-static void
-storeAufsDirCloseSwapLog(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- if (aioinfo->swaplog_fd < 0) /* not open */
- return;
- file_close(aioinfo->swaplog_fd);
- debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
- sd->index, aioinfo->swaplog_fd);
- aioinfo->swaplog_fd = -1;
- n_asyncufs_dirs--;
- assert(n_asyncufs_dirs >= 0);
- if (0 == n_asyncufs_dirs)
- safe_free(asyncufs_dir_index);
-}
-
-static void
-storeAufsDirInit(SwapDir * sd)
-{
- static int started_clean_event = 0;
- static const char *errmsg =
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.";
- storeAufsDirInitBitmap(sd);
- if (storeAufsDirVerifyCacheDirs(sd) < 0)
- fatal(errmsg);
- storeAufsDirOpenSwapLog(sd);
- storeAufsDirRebuild(sd);
- if (!started_clean_event) {
- eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, 15.0, 1);
- started_clean_event = 1;
- }
- (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize);
-}
-
-static void
-storeAufsDirRebuildFromDirectory(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
- StoreEntry *e = NULL;
- StoreEntry tmpe;
- cache_key key[MD5_DIGEST_CHARS];
- sfileno filn = 0;
- int count;
- int size;
- struct stat sb;
- int swap_hdr_len;
- int fd = -1;
- tlv *tlv_list;
- tlv *t;
- assert(rb != NULL);
- debug(47, 3) ("storeAufsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
- for (count = 0; count < rb->speed; count++) {
- assert(fd == -1);
- fd = storeAufsDirGetNextFile(rb, &filn, &size);
- if (fd == -2) {
- debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeAufsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- } else if (fd < 0) {
- continue;
- }
- assert(fd > -1);
- /* lets get file stats here */
- if (fstat(fd, &sb) < 0) {
- debug(47, 1) ("storeAufsDirRebuildFromDirectory: fstat(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFFF) == 0)
- debug(47, 3) (" %s %7d files opened so far.\n",
- rb->sd->path, rb->counts.scancount);
- debug(47, 9) ("file_in: fd=%d %08X\n", fd, filn);
- statCounter.syscalls.disk.reads++;
- if (FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
- debug(47, 1) ("storeAufsDirRebuildFromDirectory: read(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- swap_hdr_len = 0;
-#if USE_TRUNCATE
- if (sb.st_size == 0)
- continue;
-#endif
- tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
- if (tlv_list == NULL) {
- debug(47, 1) ("storeAufsDirRebuildFromDirectory: failed to get meta data\n");
- /* XXX shouldn't this be a call to storeAufsUnlink ? */
- storeAufsDirUnlinkFile(SD, filn);
- continue;
- }
- debug(47, 3) ("storeAufsDirRebuildFromDirectory: successful swap meta unpacking\n");
- memset(key, '\0', MD5_DIGEST_CHARS);
- memset(&tmpe, '\0', sizeof(StoreEntry));
- for (t = tlv_list; t; t = t->next) {
- switch (t->type) {
- case STORE_META_KEY:
- assert(t->length == MD5_DIGEST_CHARS);
- xmemcpy(key, t->value, MD5_DIGEST_CHARS);
- break;
- case STORE_META_STD:
- assert(t->length == STORE_HDR_METASIZE);
- xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
- break;
- default:
- break;
- }
- }
- storeSwapTLVFree(tlv_list);
- tlv_list = NULL;
- if (storeKeyNull(key)) {
- debug(47, 1) ("storeAufsDirRebuildFromDirectory: NULL key\n");
- storeAufsDirUnlinkFile(SD, filn);
- continue;
- }
- tmpe.hash.key = key;
- /* check sizes */
- if (tmpe.swap_file_sz == 0) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz != sb.st_size) {
- debug(47, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n",
- (long int) tmpe.swap_file_sz, (long int) sb.st_size);
- storeAufsDirUnlinkFile(SD, filn);
- continue;
- }
- if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- storeAufsDirUnlinkFile(SD, filn);
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(key);
- if (e && e->lastref >= tmpe.lastref) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (NULL != e) {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- storeRelease(e); /* release old entry */
- rb->counts.dupcount++;
- }
- rb->counts.objcount++;
- storeEntryDump(&tmpe, 5);
- e = storeAufsDirAddDiskRestore(SD, key,
- filn,
- tmpe.swap_file_sz,
- tmpe.expires,
- tmpe.timestamp,
- tmpe.lastref,
- tmpe.lastmod,
- tmpe.refcount, /* refcount */
- tmpe.flags, /* flags */
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeAufsDirRebuildFromDirectory, rb, 0.0, 1);
-}
-
-static void
-storeAufsDirRebuildFromSwapLog(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- StoreEntry *e = NULL;
- storeSwapLogData s;
- size_t ss = sizeof(storeSwapLogData);
- int count;
- int used; /* is swapfile already in use? */
- int disk_entry_newer; /* is the log entry newer than current entry? */
- double x;
- assert(rb != NULL);
- /* load a number of objects per invocation */
- for (count = 0; count < rb->speed; count++) {
- if (fread(&s, ss, 1, rb->log) != 1) {
- debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeAufsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- }
- rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
- continue;
- if (s.op >= SWAP_LOG_MAX)
- continue;
- /*
- * BC: during 2.4 development, we changed the way swap file
- * numbers are assigned and stored. The high 16 bits used
- * to encode the SD index number. There used to be a call
- * to storeDirProperFileno here that re-assigned the index
- * bits. Now, for backwards compatibility, we just need
- * to mask it off.
- */
- s.swap_filen &= 0x00FFFFFF;
- debug(47, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n",
- swap_log_op_str[(int) s.op],
- storeKeyText(s.key),
- s.swap_filen);
- if (s.op == SWAP_LOG_ADD) {
- (void) 0;
- } else if (s.op == SWAP_LOG_DEL) {
- if ((e = storeGet(s.key)) != NULL) {
- /*
- * Make sure we don't unlink the file, it might be
- * in use by a subsequent entry. Also note that
- * we don't have to subtract from store_swap_size
- * because adding to store_swap_size happens in
- * the cleanup procedure.
- */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.objcount--;
- rb->counts.cancelcount++;
- }
- continue;
- } else {
- x = log(++rb->counts.bad_log_op) / log(10.0);
- if (0.0 == x - (double) (int) x)
- debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
- rb->counts.bad_log_op);
- rb->counts.invalid++;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFF) == 0) {
- struct stat sb;
- if (0 == fstat(fileno(rb->log), &sb))
- storeRebuildProgress(SD->index,
- (int) sb.st_size / ss, rb->n_read);
- }
- if (!storeAufsDirValidFileno(SD, s.swap_filen, 0)) {
- rb->counts.invalid++;
- continue;
- }
- if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(s.key);
- used = storeAufsDirMapBitTest(SD, s.swap_filen);
- /* If this URL already exists in the cache, does the swap log
- * appear to have a newer entry? Compare 'lastref' from the
- * swap log to e->lastref. */
- disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
- /* log entry is old, ignore it */
- rb->counts.clashcount++;
- continue;
- } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
- /* swapfile taken, same URL, newer, update meta */
- if (e->store_status == STORE_OK) {
- e->lastref = s.timestamp;
- e->timestamp = s.timestamp;
- e->expires = s.expires;
- e->lastmod = s.lastmod;
- e->flags = s.flags;
- e->refcount += s.refcount;
- storeAufsDirUnrefObj(SD, e);
- } else {
- debug_trap("storeAufsDirRebuildFromSwapLog: bad condition");
- debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
- }
- continue;
- } else if (used) {
- /* swapfile in use, not by this URL, log entry is newer */
- /* This is sorta bad: the log entry should NOT be newer at this
- * point. If the log is dirty, the filesize check should have
- * caught this. If the log is clean, there should never be a
- * newer entry. */
- debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
- SD->index, s.swap_filen);
- /* I'm tempted to remove the swapfile here just to be safe,
- * but there is a bad race condition in the NOVM version if
- * the swapfile has recently been opened for writing, but
- * not yet opened for reading. Because we can't map
- * swapfiles back to StoreEntrys, we don't know the state
- * of the entry using that file. */
- /* We'll assume the existing entry is valid, probably because
- * were in a slow rebuild and the the swap file number got taken
- * and the validation procedure hasn't run. */
- assert(rb->flags.need_to_validate);
- rb->counts.clashcount++;
- continue;
- } else if (e && !disk_entry_newer) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (e) {
- /* key already exists, this swapfile not being used */
- /* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeAufsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeAufsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.dupcount++;
- } else {
- /* URL doesnt exist, swapfile not in use */
- /* load new */
- (void) 0;
- }
- /* update store_swap_size */
- rb->counts.objcount++;
- e = storeAufsDirAddDiskRestore(SD, s.key,
- s.swap_filen,
- s.swap_file_sz,
- s.expires,
- s.timestamp,
- s.lastref,
- s.lastmod,
- s.refcount,
- s.flags,
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1);
-}
-
-static int
-storeAufsDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size)
-{
- SwapDir *SD = rb->sd;
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata;
- int fd = -1;
- int used = 0;
- int dirs_opened = 0;
- debug(47, 3) ("storeAufsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
- rb->flags.init,
- rb->sd->index,
- rb->curlvl1,
- rb->curlvl2);
- if (rb->done)
- return -2;
- while (fd < 0 && rb->done == 0) {
- fd = -1;
- if (0 == rb->flags.init) { /* initialize, open first file */
- rb->done = 0;
- rb->curlvl1 = 0;
- rb->curlvl2 = 0;
- rb->in_dir = 0;
- rb->flags.init = 1;
- assert(Config.cacheSwap.n_configured > 0);
- }
- if (0 == rb->in_dir) { /* we need to read in a new directory */
- snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- rb->sd->path,
- rb->curlvl1, rb->curlvl2);
- if (dirs_opened)
- return -1;
- rb->td = opendir(rb->fullpath);
- dirs_opened++;
- if (rb->td == NULL) {
- debug(47, 1) ("storeAufsDirGetNextFile: opendir: %s: %s\n",
- rb->fullpath, xstrerror());
- } else {
- rb->entry = readdir(rb->td); /* skip . and .. */
- rb->entry = readdir(rb->td);
- if (rb->entry == NULL && errno == ENOENT)
- debug(47, 1) ("storeAufsDirGetNextFile: directory does not exist!.\n");
- debug(47, 3) ("storeAufsDirGetNextFile: Directory %s\n", rb->fullpath);
- }
- }
- if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
- rb->in_dir++;
- if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
- debug(47, 3) ("storeAufsDirGetNextFile: invalid %s\n",
- rb->entry->d_name);
- continue;
- }
- if (!storeAufsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
- debug(47, 3) ("storeAufsDirGetNextFile: %08X does not belong in %d/%d/%d\n",
- rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
- continue;
- }
- used = storeAufsDirMapBitTest(SD, rb->fn);
- if (used) {
- debug(47, 3) ("storeAufsDirGetNextFile: Locked, continuing with next.\n");
- continue;
- }
- snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
- rb->fullpath, rb->entry->d_name);
- debug(47, 3) ("storeAufsDirGetNextFile: Opening %s\n", rb->fullfilename);
- fd = file_open(rb->fullfilename, O_RDONLY | O_BINARY);
- if (fd < 0)
- debug(47, 1) ("storeAufsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
- else
- store_open_disk_fd++;
- continue;
- }
- if (rb->td != NULL)
- closedir(rb->td);
- rb->td = NULL;
- rb->in_dir = 0;
- if (++rb->curlvl2 < aioinfo->l2)
- continue;
- rb->curlvl2 = 0;
- if (++rb->curlvl1 < aioinfo->l1)
- continue;
- rb->curlvl1 = 0;
- rb->done = 1;
- }
- *filn_p = rb->fn;
- return fd;
-}
-
-/* Add a new object to the cache with empty memory copy and pointer to disk
- * use to rebuild store from disk. */
-static StoreEntry *
-storeAufsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- sfileno file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean)
-{
- StoreEntry *e = NULL;
- debug(47, 5) ("storeAufsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
- /* if you call this you'd better be sure file_number is not
- * already in use! */
- e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
- e->store_status = STORE_OK;
- storeSetMemStatus(e, NOT_IN_MEMORY);
- e->swap_status = SWAPOUT_DONE;
- e->swap_filen = file_number;
- e->swap_dirn = SD->index;
- e->swap_file_sz = swap_file_sz;
- e->lock_count = 0;
- e->lastref = lastref;
- e->timestamp = timestamp;
- e->expires = expires;
- e->lastmod = lastmod;
- e->refcount = refcount;
- e->flags = flags;
- EBIT_SET(e->flags, ENTRY_CACHABLE);
- EBIT_CLR(e->flags, RELEASE_REQUEST);
- EBIT_CLR(e->flags, KEY_PRIVATE);
- e->ping_status = PING_NONE;
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- storeAufsDirMapBitSet(SD, e->swap_filen);
- storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
- storeAufsDirReplAdd(SD, e);
- return e;
-}
-
-CBDATA_TYPE(RebuildState);
-
-static void
-storeAufsDirRebuild(SwapDir * sd)
-{
- RebuildState *rb;
- int clean = 0;
- int zero = 0;
- FILE *fp;
- EVH *func = NULL;
- CBDATA_INIT_TYPE(RebuildState);
- rb = cbdataAlloc(RebuildState);
- rb->sd = sd;
- rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
- /*
- * If the swap.state file exists in the cache_dir, then
- * we'll use storeAufsDirRebuildFromSwapLog(), otherwise we'll
- * use storeAufsDirRebuildFromDirectory() to open up each file
- * and suck in the meta data.
- */
- fp = storeAufsDirOpenTmpSwapLog(sd, &clean, &zero);
- if (fp == NULL || zero) {
- if (fp != NULL)
- fclose(fp);
- func = storeAufsDirRebuildFromDirectory;
- } else {
- func = storeAufsDirRebuildFromSwapLog;
- rb->log = fp;
- rb->flags.clean = (unsigned int) clean;
- }
- if (!clean)
- rb->flags.need_to_validate = 1;
- debug(47, 1) ("Rebuilding storage in %s (%s)\n",
- sd->path, clean ? "CLEAN" : "DIRTY");
- store_dirs_rebuilding++;
- eventAdd("storeRebuild", func, rb, 0.0, 1);
-}
-
-static void
-storeAufsDirCloseTmpSwapLog(SwapDir * sd)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
- char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new"));
- int fd;
- file_close(aioinfo->swaplog_fd);
-#if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_)
- if (unlink(swaplog_path) < 0) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeAufsDirCloseTmpSwapLog: unlink failed");
- }
-#endif
- if (xrename(new_path, swaplog_path) < 0) {
- fatal("storeAufsDirCloseTmpSwapLog: rename failed");
- }
- fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeAufsDirCloseTmpSwapLog: Failed to open swap log.");
- }
- safe_free(swaplog_path);
- safe_free(new_path);
- aioinfo->swaplog_fd = fd;
- debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
-}
-
-static FILE *
-storeAufsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
- char *clean_path = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
- char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new"));
- struct stat log_sb;
- struct stat clean_sb;
- FILE *fp;
- int fd;
- if (stat(swaplog_path, &log_sb) < 0) {
- debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return NULL;
- }
- *zero_flag = log_sb.st_size == 0 ? 1 : 0;
- /* close the existing write-only FD */
- if (aioinfo->swaplog_fd >= 0)
- file_close(aioinfo->swaplog_fd);
- /* open a write-only FD for the new log */
- fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", new_path, xstrerror());
- fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
- }
- aioinfo->swaplog_fd = fd;
- /* open a read-only stream of the old log */
- fp = fopen(swaplog_path, "rb");
- if (fp == NULL) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("Failed to open swap log for reading");
- }
- memset(&clean_sb, '\0', sizeof(struct stat));
- if (stat(clean_path, &clean_sb) < 0)
- *clean_flag = 0;
- else if (clean_sb.st_mtime < log_sb.st_mtime)
- *clean_flag = 0;
- else
- *clean_flag = 1;
- safeunlink(clean_path, 1);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return fp;
-}
-
-struct _clean_state {
- char *cur;
- char *new;
- char *cln;
- char *outbuf;
- off_t outbuf_offset;
- int fd;
- RemovalPolicyWalker *walker;
-};
-
-#define CLEAN_BUF_SZ 16384
-/*
- * Begin the process to write clean cache state. For AUFS this means
- * opening some log files and allocating write buffers. Return 0 if
- * we succeed, and assign the 'func' and 'data' return pointers.
- */
-static int
-storeAufsDirWriteCleanStart(SwapDir * sd)
-{
- struct _clean_state *state = xcalloc(1, sizeof(*state));
-#if HAVE_FCHMOD
- struct stat sb;
-#endif
- sd->log.clean.write = NULL;
- sd->log.clean.state = NULL;
- state->new = xstrdup(storeAufsDirSwapLogFile(sd, ".clean"));
- state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
- if (state->fd < 0) {
- xfree(state->new);
- xfree(state);
- return -1;
- }
- state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
- state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
- state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
- state->outbuf_offset = 0;
- state->walker = sd->repl->WalkInit(sd->repl);
- unlink(state->cln);
- debug(47, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
- state->new, state->fd);
-#if HAVE_FCHMOD
- if (stat(state->cur, &sb) == 0)
- fchmod(state->fd, sb.st_mode);
-#endif
- sd->log.clean.write = storeAufsDirWriteCleanEntry;
- sd->log.clean.state = state;
- return 0;
-}
-
-/*
- * Get the next entry that is a candidate for clean log writing
- */
-const StoreEntry *
-storeAufsDirCleanLogNextEntry(SwapDir * sd)
-{
- const StoreEntry *entry = NULL;
- struct _clean_state *state = sd->log.clean.state;
- if (state->walker)
- entry = state->walker->Next(state->walker);
- return entry;
-}
-
-/*
- * "write" an entry to the clean log file.
- */
-static void
-storeAufsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
-{
- storeSwapLogData s;
- static size_t ss = sizeof(storeSwapLogData);
- struct _clean_state *state = sd->log.clean.state;
- memset(&s, '\0', ss);
- s.op = (char) SWAP_LOG_ADD;
- s.swap_filen = e->swap_filen;
- s.timestamp = e->timestamp;
- s.lastref = e->lastref;
- s.expires = e->expires;
- s.lastmod = e->lastmod;
- s.swap_file_sz = e->swap_file_sz;
- s.refcount = e->refcount;
- s.flags = e->flags;
- xmemcpy(&s.key, e->hash.key, MD5_DIGEST_CHARS);
- xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
- state->outbuf_offset += ss;
- /* buffered write */
- if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
- return;
- }
- state->outbuf_offset = 0;
- }
-}
-
-static void
-storeAufsDirWriteCleanDone(SwapDir * sd)
-{
- int fd;
- struct _clean_state *state = sd->log.clean.state;
- if (NULL == state)
- return;
- if (state->fd < 0)
- return;
- state->walker->Done(state->walker);
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile "
- "not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- }
- safe_free(state->outbuf);
- /*
- * You can't rename open files on Microsoft "operating systems"
- * so we have to close before renaming.
- */
- storeAufsDirCloseSwapLog(sd);
- /* save the fd value for a later test */
- fd = state->fd;
- /* rename */
- if (state->fd >= 0) {
-#if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_)
- file_close(state->fd);
- state->fd = -1;
- if (unlink(state->cur) < 0)
- debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
- xstrerror(), state->cur);
-#endif
- xrename(state->new, state->cur);
- }
- /* touch a timestamp file if we're not still validating */
- if (store_dirs_rebuilding)
- (void) 0;
- else if (fd < 0)
- (void) 0;
- else
- file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
- /* close */
- safe_free(state->cur);
- safe_free(state->new);
- safe_free(state->cln);
- if (state->fd >= 0)
- file_close(state->fd);
- state->fd = -1;
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
-}
-
-static void
-storeSwapLogDataFree(void *s)
-{
- memFree(s, MEM_SWAP_LOG_DATA);
-}
-
-static void
-storeAufsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) sd->fsdata;
- storeSwapLogData *s = memAllocate(MEM_SWAP_LOG_DATA);
- s->op = (char) op;
- s->swap_filen = e->swap_filen;
- s->timestamp = e->timestamp;
- s->lastref = e->lastref;
- s->expires = e->expires;
- s->lastmod = e->lastmod;
- s->swap_file_sz = e->swap_file_sz;
- s->refcount = e->refcount;
- s->flags = e->flags;
- xmemcpy(s->key, e->hash.key, MD5_DIGEST_CHARS);
- file_write(aioinfo->swaplog_fd,
- -1,
- s,
- sizeof(storeSwapLogData),
- NULL,
- NULL,
- (FREE *) storeSwapLogDataFree);
-}
-
-static void
-storeAufsDirNewfs(SwapDir * sd)
-{
- debug(47, 3) ("Creating swap space in %s\n", sd->path);
- storeAufsDirCreateDirectory(sd->path, 0);
- storeAufsDirCreateSwapSubDirs(sd);
-}
-
-static int
-rev_int_sort(const void *A, const void *B)
-{
- const int *i1 = A;
- const int *i2 = B;
- return *i2 - *i1;
-}
-
-static int
-storeAufsDirClean(int swap_index)
-{
- DIR *dp = NULL;
- struct dirent *de = NULL;
- LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
- LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
-#if USE_TRUNCATE
- struct stat sb;
-#endif
- int files[20];
- int swapfileno;
- int fn; /* same as swapfileno, but with dirn bits set */
- int n = 0;
- int k = 0;
- int N0, N1, N2;
- int D0, D1, D2;
- SwapDir *SD;
- squidaioinfo_t *aioinfo;
- N0 = n_asyncufs_dirs;
- D0 = asyncufs_dir_index[swap_index % N0];
- SD = &Config.cacheSwap.swapDirs[D0];
- aioinfo = (squidaioinfo_t *) SD->fsdata;
- N1 = aioinfo->l1;
- D1 = (swap_index / N0) % N1;
- N2 = aioinfo->l2;
- D2 = ((swap_index / N0) / N1) % N2;
- snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- Config.cacheSwap.swapDirs[D0].path, D1, D2);
- debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
- dp = opendir(p1);
- if (dp == NULL) {
- if (errno == ENOENT) {
- debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
- if (mkdir(p1, 0777) == 0)
- return 0;
- }
- debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
- safeunlink(p1, 1);
- return 0;
- }
- while ((de = readdir(dp)) != NULL && k < 20) {
- if (sscanf(de->d_name, "%X", &swapfileno) != 1)
- continue;
- fn = swapfileno; /* XXX should remove this cruft ! */
- if (storeAufsDirValidFileno(SD, fn, 1))
- if (storeAufsDirMapBitTest(SD, fn))
- if (storeAufsFilenoBelongsHere(fn, D0, D1, D2))
- continue;
-#if USE_TRUNCATE
- if (!stat(de->d_name, &sb))
- if (sb.st_size == 0)
- continue;
-#endif
- files[k++] = swapfileno;
- }
- closedir(dp);
- if (k == 0)
- return 0;
- qsort(files, k, sizeof(int), rev_int_sort);
- if (k > 10)
- k = 10;
- for (n = 0; n < k; n++) {
- debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
- snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
-#if USE_TRUNCATE
- truncate(p2, 0);
-#else
- safeunlink(p2, 0);
-#endif
- statCounter.swap.files_cleaned++;
- }
- debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
- return k;
-}
-
-static void
-storeAufsDirCleanEvent(void *unused)
-{
- static int swap_index = 0;
- int i;
- int j = 0;
- int n = 0;
- /*
- * Assert that there are AUFS cache_dirs configured, otherwise
- * we should never be called.
- */
- assert(n_asyncufs_dirs);
- if (NULL == asyncufs_dir_index) {
- SwapDir *sd;
- squidaioinfo_t *aioinfo;
- /*
- * Initialize the little array that translates AUFS cache_dir
- * number into the Config.cacheSwap.swapDirs array index.
- */
- asyncufs_dir_index = xcalloc(n_asyncufs_dirs, sizeof(*asyncufs_dir_index));
- for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
- sd = &Config.cacheSwap.swapDirs[i];
- if (!storeAufsDirIs(sd))
- continue;
- asyncufs_dir_index[n++] = i;
- aioinfo = (squidaioinfo_t *) sd->fsdata;
- j += (aioinfo->l1 * aioinfo->l2);
- }
- assert(n == n_asyncufs_dirs);
- /*
- * Start the storeAufsDirClean() swap_index with a random
- * value. j equals the total number of AUFS level 2
- * swap directories
- */
- swap_index = (int) (squid_random() % j);
- }
- if (0 == store_dirs_rebuilding) {
- n = storeAufsDirClean(swap_index);
- swap_index++;
- }
- eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL,
- 15.0 * exp(-0.25 * n), 1);
-}
-
-static int
-storeAufsDirIs(SwapDir * sd)
-{
- if (strncmp(sd->type, "aufs", 4) == 0)
- return 1;
- return 0;
-}
-
-/*
- * Does swapfile number 'fn' belong in cachedir #F0,
- * level1 dir #F1, level2 dir #F2?
- */
-static int
-storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2)
-{
- int D1, D2;
- int L1, L2;
- int filn = fn;
- squidaioinfo_t *aioinfo;
- assert(F0 < Config.cacheSwap.n_configured);
- aioinfo = (squidaioinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata;
- L1 = aioinfo->l1;
- L2 = aioinfo->l2;
- D1 = ((filn / L2) / L2) % L1;
- if (F1 != D1)
- return 0;
- D2 = (filn / L2) % L2;
- if (F2 != D2)
- return 0;
- return 1;
-}
-
-int
-storeAufsDirValidFileno(SwapDir * SD, sfileno filn, int flag)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata;
- if (filn < 0)
- return 0;
- /*
- * If flag is set it means out-of-range file number should
- * be considered invalid.
- */
- if (flag)
- if (filn > aioinfo->map->max_n_files)
- return 0;
- return 1;
-}
-
-void
-storeAufsDirMaintain(SwapDir * SD)
-{
- StoreEntry *e = NULL;
- int removed = 0;
- int max_scan;
- int max_remove;
- double f;
- RemovalPurgeWalker *walker;
- /* We can't delete objects while rebuilding swap */
- if (store_dirs_rebuilding) {
- return;
- } else {
- f = (double) (SD->cur_size - SD->low_size) / (SD->max_size - SD->low_size);
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
- max_scan = (int) (f * 400.0 + 100.0);
- max_remove = (int) (f * 70.0 + 10.0);
- /*
- * This is kinda cheap, but so we need this priority hack?
- */
- }
- debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n",
- f, max_scan, max_remove);
- walker = SD->repl->PurgeInit(SD->repl, max_scan);
- while (1) {
- if (SD->cur_size < SD->low_size)
- break;
- if (removed >= max_remove)
- break;
- e = walker->Next(walker);
- if (!e)
- break; /* no more objects */
- removed++;
- storeRelease(e);
- }
- walker->Done(walker);
- debug(47, (removed ? 2 : 3)) ("storeAufsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
- SD->path, removed, max_remove, f, max_scan);
-}
-
/*
* storeAufsDirCheckObj
*
return loadav;
}
-/*
- * storeAufsDirRefObj
- *
- * This routine is called whenever an object is referenced, so we can
- * maintain replacement information within the storage fs.
- */
-void
-storeAufsDirRefObj(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 3) ("storeAufsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
- e->swap_filen);
- if (SD->repl->Referenced)
- SD->repl->Referenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeAufsDirUnrefObj
- * This routine is called whenever the last reference to an object is
- * removed, to maintain replacement information within the storage fs.
- */
-void
-storeAufsDirUnrefObj(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 3) ("storeAufsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
- e->swap_filen);
- if (SD->repl->Dereferenced)
- SD->repl->Dereferenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeAufsDirUnlinkFile
- *
- * This routine unlinks a file and pulls it out of the bitmap.
- * It used to be in storeAufsUnlink(), however an interface change
- * forced this bit of code here. Eeek.
- */
void
-storeAufsDirUnlinkFile(SwapDir * SD, sfileno f)
+storeAufsDirIOUnlinkFile(char *path)
{
- debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f);
- /* storeAufsDirMapBitReset(SD, f); */
#if USE_TRUNCATE_NOT_UNLINK
- aioTruncate(storeAufsDirFullPath(SD, f, NULL), NULL, NULL);
+ aioTruncate(path, NULL, NULL);
#else
- aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL);
+ aioUnlink(path, NULL, NULL);
#endif
}
-/*
- * Add and remove the given StoreEntry from the replacement policy in
- * use.
- */
-
-void
-storeAufsDirReplAdd(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 4) ("storeAufsDirReplAdd: added node %p to dir %d\n", e,
- SD->index);
- SD->repl->Add(SD->repl, e, &e->repl);
-}
-
-
-void
-storeAufsDirReplRemove(StoreEntry * e)
-{
- SwapDir *SD = INDEXSD(e->swap_dirn);
- debug(47, 4) ("storeAufsDirReplRemove: remove node %p from dir %d\n", e,
- SD->index);
- SD->repl->Remove(SD->repl, e, &e->repl);
-}
-
-
-
/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
-void
-storeAufsDirStats(SwapDir * SD, StoreEntry * sentry)
-{
- squidaioinfo_t *aioinfo = SD->fsdata;
- int totl_kb = 0;
- int free_kb = 0;
- int totl_in = 0;
- int free_in = 0;
- int x;
- storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1);
- storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2);
- storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
- storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
- storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
- 100.0 * SD->cur_size / SD->max_size);
- storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
- aioinfo->map->n_files_in_map, aioinfo->map->max_n_files,
- percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files));
- x = storeDirGetUFSStats(SD->path, &totl_kb, &free_kb, &totl_in, &free_in);
- if (0 == x) {
- storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
- totl_kb - free_kb,
- totl_kb,
- percent(totl_kb - free_kb, totl_kb));
- storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
- totl_in - free_in,
- totl_in,
- percent(totl_in - free_in, totl_in));
- }
- storeAppendPrintf(sentry, "Flags:");
- if (SD->flags.selected)
- storeAppendPrintf(sentry, " SELECTED");
- if (SD->flags.read_only)
- storeAppendPrintf(sentry, " READ-ONLY");
- storeAppendPrintf(sentry, "\n");
-}
-
static struct cache_dir_option options[] =
{
#if NOT_YET_DONE
void
storeAufsDirDump(StoreEntry * entry, SwapDir * s)
{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) s->fsdata;
- storeAppendPrintf(entry, " %d %d %d",
- s->max_size >> 10,
- aioinfo->l1,
- aioinfo->l2);
+ commonUfsDirDump (entry, s);
dump_cachedir_options(entry, options, s);
}
-/*
- * Only "free" the filesystem specific stuff here
- */
-static void
-storeAufsDirFree(SwapDir * s)
-{
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) s->fsdata;
- if (aioinfo->swaplog_fd > -1) {
- file_close(aioinfo->swaplog_fd);
- aioinfo->swaplog_fd = -1;
- }
- filemapFreeMemory(aioinfo->map);
- xfree(aioinfo);
- s->fsdata = NULL; /* Will aid debugging... */
-}
-
-char *
-storeAufsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath)
-{
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- squidaioinfo_t *aioinfo = (squidaioinfo_t *) SD->fsdata;
- int L1 = aioinfo->l1;
- int L2 = aioinfo->l2;
- if (!fullpath)
- fullpath = fullfilename;
- fullpath[0] = '\0';
- snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
- SD->path,
- ((filn / L2) / L2) % L1,
- (filn / L2) % L2,
- filn);
- return fullpath;
-}
-
-/*
- * storeAufsCleanupDoubleCheck
- *
- * This is called by storeCleanup() if -S was given on the command line.
- */
-static int
-storeAufsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
-{
- struct stat sb;
- if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(47, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(47, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
- storeAufsDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
- }
- if (e->swap_file_sz != sb.st_size) {
- debug(47, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(47, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
- storeAufsDirFullPath(sd, e->swap_filen, NULL));
- debug(47, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %ld, FILE SIZE: %ld\n",
- (long int) e->swap_file_sz, (long int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
- }
- return 0;
-}
-
/*
* storeAufsDirParse *
* Called when a *new* fs is being setup.
int size;
int l1;
int l2;
- squidaioinfo_t *aioinfo;
+ squidufsinfo_t *aioinfo;
i = GetInteger();
size = i << 10; /* Mbytes to kbytes */
if (l2 <= 0)
fatal("storeAufsDirParse: invalid level 2 directories value");
- aioinfo = xmalloc(sizeof(squidaioinfo_t));
+ aioinfo = xmalloc(sizeof(squidufsinfo_t));
if (aioinfo == NULL)
- fatal("storeAufsDirParse: couldn't xmalloc() squidaioinfo_t!\n");
+ fatal("storeAufsDirParse: couldn't xmalloc() squidufsinfo_t!\n");
sd->index = index;
sd->path = xstrdup(path);
aioinfo->swaplog_fd = -1;
aioinfo->map = NULL; /* Debugging purposes */
aioinfo->suggest = 0;
- sd->init = storeAufsDirInit;
- sd->newfs = storeAufsDirNewfs;
+ aioinfo->io.storeDirUnlinkFile = storeAufsDirIOUnlinkFile;
+ sd->init = commonUfsDirInit;
+ sd->newfs = commonUfsDirNewfs;
sd->dump = storeAufsDirDump;
- sd->freefs = storeAufsDirFree;
- sd->dblcheck = storeAufsCleanupDoubleCheck;
- sd->statfs = storeAufsDirStats;
- sd->maintainfs = storeAufsDirMaintain;
+ sd->freefs = commonUfsDirFree;
+ sd->dblcheck = commonUfsCleanupDoubleCheck;
+ sd->statfs = commonUfsDirStats;
+ sd->maintainfs = commonUfsDirMaintain;
sd->checkobj = storeAufsDirCheckObj;
- sd->refobj = storeAufsDirRefObj;
- sd->unrefobj = storeAufsDirUnrefObj;
+ sd->refobj = commonUfsDirRefObj;
+ sd->unrefobj = commonUfsDirUnrefObj;
sd->callback = aioCheckCallbacks;
sd->sync = aioSync;
sd->obj.create = storeAufsCreate;
sd->obj.read = storeAufsRead;
sd->obj.write = storeAufsWrite;
sd->obj.unlink = storeAufsUnlink;
- sd->log.open = storeAufsDirOpenSwapLog;
- sd->log.close = storeAufsDirCloseSwapLog;
- sd->log.write = storeAufsDirSwapLog;
- sd->log.clean.start = storeAufsDirWriteCleanStart;
- sd->log.clean.nextentry = storeAufsDirCleanLogNextEntry;
- sd->log.clean.done = storeAufsDirWriteCleanDone;
+ sd->log.open = commonUfsDirOpenSwapLog;
+ sd->log.close = commonUfsDirCloseSwapLog;
+ sd->log.write = commonUfsDirSwapLog;
+ sd->log.clean.start = commonUfsDirWriteCleanStart;
+ sd->log.clean.nextentry = commonUfsDirCleanLogNextEntry;
+ sd->log.clean.done = commonUfsDirWriteCleanDone;
parse_cachedir_options(sd, options, 0);
#include "squid.h"
#include "store_asyncufs.h"
+#include "ufscommon.h"
#if ASYNC_READ
static AIOCB storeAufsReadDone;
STIOCB * callback, void *callback_data)
{
sfileno f = e->swap_filen;
- char *path = storeAufsDirFullPath(SD, f, NULL);
+ char *path = commonUfsDirFullPath(SD, f, NULL);
storeIOState *sio;
#if !ASYNC_OPEN
int fd;
/* Allocate a number */
dirn = SD->index;
- filn = storeAufsDirMapBitAllocate(SD);
- path = storeAufsDirFullPath(SD, filn, NULL);
+ filn = commonUfsDirMapBitAllocate(SD);
+ path = commonUfsDirFullPath(SD, filn, NULL);
debug(79, 3) ("storeAufsCreate: fileno %08X\n", filn);
/*
#endif
/* now insert into the replacement policy */
- storeAufsDirReplAdd(SD, e);
+ commonUfsDirReplAdd(SD, e);
return sio;
}
storeAufsUnlink(SwapDir * SD, StoreEntry * e)
{
debug(79, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen);
- storeAufsDirReplRemove(e);
- storeAufsDirMapBitReset(SD, e->swap_filen);
- storeAufsDirUnlinkFile(SD, e->swap_filen);
+ commonUfsDirReplRemove(e);
+ commonUfsDirMapBitReset(SD, e->swap_filen);
+ commonUfsDirUnlinkFile(SD, e->swap_filen);
}
/* === STATIC =========================================================== */
if (errflag || fd < 0) {
errno = errflag;
debug(79, 0) ("storeAufsOpenDone: %s\n", xstrerror());
- debug(79, 1) ("\t%s\n", storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
+ debug(79, 1) ("\t%s\n", commonUfsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
storeAufsIOCallback(sio, DISK_ERROR);
return;
}
store_open_disk_fd++;
aiostate->fd = fd;
commSetCloseOnExec(fd);
- fd_open(fd, FD_FILE, storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
+ fd_open(fd, FD_FILE, commonUfsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
if (FILE_MODE(sio->mode) == O_WRONLY) {
if (storeAufsKickWriteQueue(sio))
return;
/*
- * $Id: store_dir_diskd.cc,v 1.70 2002/08/09 10:57:46 robertc Exp $
+ * $Id: store_dir_diskd.cc,v 1.71 2002/10/12 09:45:57 robertc Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
#include "store_diskd.h"
-#define DefaultLevelOneDirs 16
-#define DefaultLevelTwoDirs 256
-#define STORE_META_BDISKDZ 4096
+#include "ufscommon.h"
diskd_stats_t diskd_stats;
-typedef struct _RebuildState RebuildState;
-struct _RebuildState {
- SwapDir *sd;
- int n_read;
- FILE *log;
- int speed;
- int curlvl1;
- int curlvl2;
- struct {
- unsigned int need_to_validate:1;
- unsigned int clean:1;
- unsigned int init:1;
- } flags;
- int done;
- int in_dir;
- int fn;
- struct dirent *entry;
- DIR *td;
- char fullpath[SQUID_MAXPATHLEN];
- char fullfilename[SQUID_MAXPATHLEN];
- struct _store_rebuild_data counts;
-};
-
-static int n_diskd_dirs = 0;
-static int *diskd_dir_index = NULL;
MemPool *diskd_state_pool = NULL;
static int diskd_initialised = 0;
-static char *storeDiskdDirSwapSubDir(SwapDir *, int subdirn);
-static int storeDiskdDirCreateDirectory(const char *path, int);
-static int storeDiskdDirVerifyCacheDirs(SwapDir *);
-static int storeDiskdDirVerifyDirectory(const char *path);
-static void storeDiskdDirCreateSwapSubDirs(SwapDir *);
-static char *storeDiskdDirSwapLogFile(SwapDir *, const char *);
-static EVH storeDiskdDirRebuildFromDirectory;
-static EVH storeDiskdDirRebuildFromSwapLog;
-static int storeDiskdDirGetNextFile(RebuildState *, sfileno *, int *size);
-static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- sfileno file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean);
-static void storeDiskdDirRebuild(SwapDir * sd);
-static void storeDiskdDirCloseTmpSwapLog(SwapDir * sd);
-static FILE *storeDiskdDirOpenTmpSwapLog(SwapDir *, int *, int *);
-static STLOGOPEN storeDiskdDirOpenSwapLog;
static STINIT storeDiskdDirInit;
-static STFREE storeDiskdDirFree;
-static STLOGCLEANSTART storeDiskdDirWriteCleanStart;
-static STLOGCLEANNEXTENTRY storeDiskdDirCleanLogNextEntry;
-static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry;
-static STLOGCLEANDONE storeDiskdDirWriteCleanDone;
-static STLOGCLOSE storeDiskdDirCloseSwapLog;
-static STLOGWRITE storeDiskdDirSwapLog;
-static STNEWFS storeDiskdDirNewfs;
static STDUMP storeDiskdDirDump;
-static STMAINTAINFS storeDiskdDirMaintain;
static STCHECKOBJ storeDiskdDirCheckObj;
-static STREFOBJ storeDiskdDirRefObj;
-static STUNREFOBJ storeDiskdDirUnrefObj;
-static QS rev_int_sort;
-static int storeDiskdDirClean(int swap_index);
-static EVH storeDiskdDirCleanEvent;
-static int storeDiskdDirIs(SwapDir * sd);
-static int storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2);
-static int storeDiskdCleanupDoubleCheck(SwapDir *, StoreEntry *);
static void storeDiskdDirStats(SwapDir *, StoreEntry *);
-static void storeDiskdDirInitBitmap(SwapDir *);
-static int storeDiskdDirValidFileno(SwapDir *, sfileno, int);
static void storeDiskdStats(StoreEntry * sentry);
static void storeDiskdDirSync(SwapDir *);
+static void storeDiskdDirIOUnlinkFile(char *path);
/* The only externally visible interface */
STSETUP storeFsSetup_diskd;
-/*
- * These functions were ripped straight out of the heart of store_dir.c.
- * They assume that the given filenum is on a diskd partiton, which may or
- * may not be true..
- * XXX this evilness should be tidied up at a later date!
- */
-
-static int
-storeDiskdDirMapBitTest(SwapDir * SD, sfileno filn)
-{
- diskdinfo_t *diskdinfo;
- diskdinfo = SD->fsdata;
- return file_map_bit_test(diskdinfo->map, filn);
-}
-
-static void
-storeDiskdDirMapBitSet(SwapDir * SD, sfileno filn)
-{
- diskdinfo_t *diskdinfo;
- diskdinfo = SD->fsdata;
- file_map_bit_set(diskdinfo->map, filn);
-}
-
-void
-storeDiskdDirMapBitReset(SwapDir * SD, sfileno filn)
-{
- diskdinfo_t *diskdinfo;
- diskdinfo = SD->fsdata;
- /*
- * We have to test the bit before calling file_map_bit_reset.
- * file_map_bit_reset doesn't do bounds checking. It assumes
- * filn is a valid file number, but it might not be because
- * the map is dynamic in size. Also clearing an already clear
- * bit puts the map counter of-of-whack.
- */
- if (file_map_bit_test(diskdinfo->map, filn))
- file_map_bit_reset(diskdinfo->map, filn);
-}
-
-int
-storeDiskdDirMapBitAllocate(SwapDir * SD)
-{
- diskdinfo_t *diskdinfo = SD->fsdata;
- int fn;
- fn = file_map_allocate(diskdinfo->map, diskdinfo->suggest);
- file_map_bit_set(diskdinfo->map, fn);
- diskdinfo->suggest = fn + 1;
- return fn;
-}
-
-/*
- * Initialise the diskd bitmap
- *
- * If there already is a bitmap, and the numobjects is larger than currently
- * configured, we allocate a new bitmap and 'grow' the old one into it.
- */
-static void
-storeDiskdDirInitBitmap(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
-
- if (diskdinfo->map == NULL) {
- /* First time */
- diskdinfo->map = file_map_create();
- } else if (diskdinfo->map->max_n_files) {
- /* it grew, need to expand */
- /* XXX We don't need it anymore .. */
- }
- /* else it shrunk, and we leave the old one in place */
-}
-
-static char *
-storeDiskdDirSwapSubDir(SwapDir * sd, int subdirn)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
-
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- assert(0 <= subdirn && subdirn < diskdinfo->l1);
- snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
- return fullfilename;
-}
-
-static int
-storeDiskdDirCreateDirectory(const char *path, int should_exist)
-{
- int created = 0;
- struct stat st;
- getCurrentTime();
- if (0 == stat(path, &st)) {
- if (S_ISDIR(st.st_mode)) {
- debug(47, should_exist ? 3 : 1) ("%s exists\n", path);
- } else {
- fatalf("Swap directory %s is not a directory.", path);
- }
- } else if (0 == mkdir(path, 0755)) {
- debug(47, should_exist ? 1 : 3) ("%s created\n", path);
- created = 1;
- } else {
- fatalf("Failed to make swap directory %s: %s",
- path, xstrerror());
- }
- return created;
-}
-
-static int
-storeDiskdDirVerifyDirectory(const char *path)
-{
- struct stat sb;
- if (stat(path, &sb) < 0) {
- debug(47, 0) ("%s: %s\n", path, xstrerror());
- return -1;
- }
- if (S_ISDIR(sb.st_mode) == 0) {
- debug(47, 0) ("%s is not a directory\n", path);
- return -1;
- }
- return 0;
-}
-
-/*
- * This function is called by storeDiskdDirInit(). If this returns < 0,
- * then Squid exits, complains about swap directories not
- * existing, and instructs the admin to run 'squid -z'
- */
-static int
-storeDiskdDirVerifyCacheDirs(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- int j;
- const char *path = sd->path;
-
- if (storeDiskdDirVerifyDirectory(path) < 0)
- return -1;
- for (j = 0; j < diskdinfo->l1; j++) {
- path = storeDiskdDirSwapSubDir(sd, j);
- if (storeDiskdDirVerifyDirectory(path) < 0)
- return -1;
- }
- return 0;
-}
-
-static void
-storeDiskdDirCreateSwapSubDirs(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- int i, k;
- int should_exist;
- LOCAL_ARRAY(char, name, MAXPATHLEN);
- for (i = 0; i < diskdinfo->l1; i++) {
- snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
- if (storeDiskdDirCreateDirectory(name, 0))
- should_exist = 0;
- else
- should_exist = 1;
- debug(47, 1) ("Making directories in %s\n", name);
- for (k = 0; k < diskdinfo->l2; k++) {
- snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
- storeDiskdDirCreateDirectory(name, should_exist);
- }
- }
-}
-
-static char *
-storeDiskdDirSwapLogFile(SwapDir * sd, const char *ext)
-{
- LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, digit, 32);
- char *pathtmp2;
- if (Config.Log.swap) {
- xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
- pathtmp2 = pathtmp;
- while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
- *pathtmp2 = '.';
- while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
- pathtmp[strlen(pathtmp) - 1] = '\0';
- for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
- snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
- if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
- strcat(path, ".");
- snprintf(digit, 32, "%02d", sd->index);
- strncat(path, digit, 3);
- }
- } else {
- xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
- strcat(path, "/swap.state");
- }
- if (ext)
- strncat(path, ext, 16);
- return path;
-}
-
-static void
-storeDiskdDirOpenSwapLog(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- char *path;
- int fd;
- path = storeDiskdDirSwapLogFile(sd, NULL);
- fd = file_open(path, O_WRONLY | O_CREAT);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", path, xstrerror());
- fatal("storeDiskdDirOpenSwapLog: Failed to open swap log.");
- }
- debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
- diskdinfo->swaplog_fd = fd;
- if (0 == n_diskd_dirs)
- assert(NULL == diskd_dir_index);
- n_diskd_dirs++;
- assert(n_diskd_dirs <= Config.cacheSwap.n_configured);
-}
-
-static void
-storeDiskdDirCloseSwapLog(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- if (diskdinfo->swaplog_fd < 0) /* not open */
- return;
- file_close(diskdinfo->swaplog_fd);
- debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
- sd->index, diskdinfo->swaplog_fd);
- diskdinfo->swaplog_fd = -1;
- n_diskd_dirs--;
- assert(n_diskd_dirs >= 0);
- if (0 == n_diskd_dirs)
- safe_free(diskd_dir_index);
-}
-
static void
storeDiskdDirInit(SwapDir * sd)
{
- static int started_clean_event = 0;
int x;
int i;
int rfd;
char skey2[32];
char skey3[32];
diskdinfo_t *diskdinfo = sd->fsdata;
- static const char *errmsg =
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.";
ikey = (getpid() << 10) + (sd->index << 2);
ikey &= 0x7fffffff;
fd_note(diskdinfo->wfd, "squid -> diskd");
commSetTimeout(diskdinfo->wfd, -1, NULL, NULL);
commSetNonBlocking(diskdinfo->wfd);
- storeDiskdDirInitBitmap(sd);
- if (storeDiskdDirVerifyCacheDirs(sd) < 0)
- fatal(errmsg);
- storeDiskdDirOpenSwapLog(sd);
- storeDiskdDirRebuild(sd);
- if (!started_clean_event) {
- eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, 15.0, 1);
- started_clean_event = 1;
- }
- (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize);
+
+ commonUfsDirInit (sd);
+
comm_quick_poll_required();
}
return retval;
}
-
-
-static void
-storeDiskdDirRebuildFromDirectory(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
- StoreEntry *e = NULL;
- StoreEntry tmpe;
- cache_key key[MD5_DIGEST_CHARS];
- sfileno filn = 0;
- int count;
- int size;
- struct stat sb;
- int swap_hdr_len;
- int fd = -1;
- tlv *tlv_list;
- tlv *t;
- assert(rb != NULL);
- debug(47, 3) ("storeDiskdDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
- for (count = 0; count < rb->speed; count++) {
- assert(fd == -1);
- fd = storeDiskdDirGetNextFile(rb, &filn, &size);
- if (fd == -2) {
- debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeDiskdDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- } else if (fd < 0) {
- continue;
- }
- assert(fd > -1);
- /* lets get file stats here */
- if (fstat(fd, &sb) < 0) {
- debug(47, 1) ("storeDiskdDirRebuildFromDirectory: fstat(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFFF) == 0)
- debug(47, 3) (" %s %7d files opened so far.\n",
- rb->sd->path, rb->counts.scancount);
- debug(47, 9) ("file_in: fd=%d %08X\n", fd, filn);
- statCounter.syscalls.disk.reads++;
- if (FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
- debug(47, 1) ("storeDiskdDirRebuildFromDirectory: read(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- swap_hdr_len = 0;
-#if USE_TRUNCATE
- if (sb.st_size == 0)
- continue;
-#endif
- tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
- if (tlv_list == NULL) {
- debug(47, 1) ("storeDiskdDirRebuildFromDirectory: failed to get meta data\n");
- /* XXX shouldn't this be a call to storeDiskdUnlink ? */
- storeDiskdDirUnlinkFile(SD, filn);
- continue;
- }
- debug(47, 3) ("storeDiskdDirRebuildFromDirectory: successful swap meta unpacking\n");
- memset(key, '\0', MD5_DIGEST_CHARS);
- memset(&tmpe, '\0', sizeof(StoreEntry));
- for (t = tlv_list; t; t = t->next) {
- switch (t->type) {
- case STORE_META_KEY:
- assert(t->length == MD5_DIGEST_CHARS);
- xmemcpy(key, t->value, MD5_DIGEST_CHARS);
- break;
- case STORE_META_STD:
- assert(t->length == STORE_HDR_METASIZE);
- xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
- break;
- default:
- break;
- }
- }
- storeSwapTLVFree(tlv_list);
- tlv_list = NULL;
- if (storeKeyNull(key)) {
- debug(47, 1) ("storeDiskdDirRebuildFromDirectory: NULL key\n");
- storeDiskdDirUnlinkFile(SD, filn);
- continue;
- }
- tmpe.hash.key = key;
- /* check sizes */
- if (tmpe.swap_file_sz == 0) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz != sb.st_size) {
- debug(47, 1) ("storeDiskdDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n",
- (long int) tmpe.swap_file_sz, (long int) sb.st_size);
- storeDiskdDirUnlinkFile(SD, filn);
- continue;
- }
- if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- storeDiskdDirUnlinkFile(SD, filn);
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(key);
- if (e && e->lastref >= tmpe.lastref) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (NULL != e) {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- storeRelease(e); /* release old entry */
- rb->counts.dupcount++;
- }
- rb->counts.objcount++;
- storeEntryDump(&tmpe, 5);
- e = storeDiskdDirAddDiskRestore(SD, key,
- filn,
- tmpe.swap_file_sz,
- tmpe.expires,
- tmpe.timestamp,
- tmpe.lastref,
- tmpe.lastmod,
- tmpe.refcount, /* refcount */
- tmpe.flags, /* flags */
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeDiskdDirRebuildFromDirectory, rb, 0.0, 1);
-}
-
-static void
-storeDiskdDirRebuildFromSwapLog(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- StoreEntry *e = NULL;
- storeSwapLogData s;
- size_t ss = sizeof(storeSwapLogData);
- int count;
- int used; /* is swapfile already in use? */
- int disk_entry_newer; /* is the log entry newer than current entry? */
- double x;
- assert(rb != NULL);
- /* load a number of objects per invocation */
- for (count = 0; count < rb->speed; count++) {
- if (fread(&s, ss, 1, rb->log) != 1) {
- debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeDiskdDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- }
- rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
- continue;
- if (s.op >= SWAP_LOG_MAX)
- continue;
- /*
- * BC: during 2.4 development, we changed the way swap file
- * numbers are assigned and stored. The high 16 bits used
- * to encode the SD index number. There used to be a call
- * to storeDirProperFileno here that re-assigned the index
- * bits. Now, for backwards compatibility, we just need
- * to mask it off.
- */
- s.swap_filen &= 0x00FFFFFF;
- debug(47, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n",
- swap_log_op_str[(int) s.op],
- storeKeyText(s.key),
- s.swap_filen);
- if (s.op == SWAP_LOG_ADD) {
- /*
- * Here we have some special checks for large files.
- * I've been seeing a system crash followed by a reboot
- * that seems to corrupt the swap log. Squid believes
- * that the disk holds some really large files. It
- * complains about using being over the high water mark
- * and proceeds to delete files as fast as it can. To
- * prevent that, we call stat() on sufficiently large
- * files (>128KB) and reject those that are missing or
- * have the wrong size.
- */
- struct stat sb;
- char *p = storeDiskdDirFullPath(SD, s.swap_filen, NULL);
- if (s.swap_file_sz < (1 << 17)) {
- (void) 0;
- } else if (stat(p, &sb) < 0) {
- debug(47, 2) ("its missing!: %s\n", p);
- continue;
- } else if (sb.st_size != s.swap_file_sz) {
- debug(47, 2) ("size mismatch!: stat=%d, log=%d\n",
- (int) sb.st_size, (int) s.swap_file_sz);
- continue;
- } else {
- debug(47, 2) ("big file (%d bytes) checks out\n",
- (int) s.swap_file_sz);
- }
- } else if (s.op == SWAP_LOG_DEL) {
- if ((e = storeGet(s.key)) != NULL) {
- /*
- * Make sure we don't unlink the file, it might be
- * in use by a subsequent entry. Also note that
- * we don't have to subtract from store_swap_size
- * because adding to store_swap_size happens in
- * the cleanup procedure.
- */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.objcount--;
- rb->counts.cancelcount++;
- }
- continue;
- } else {
- x = log(++rb->counts.bad_log_op) / log(10.0);
- if (0.0 == x - (double) (int) x)
- debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
- rb->counts.bad_log_op);
- rb->counts.invalid++;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFF) == 0) {
- struct stat sb;
- if (0 == fstat(fileno(rb->log), &sb))
- storeRebuildProgress(SD->index,
- (int) sb.st_size / ss, rb->n_read);
- }
- if (!storeDiskdDirValidFileno(SD, s.swap_filen, 0)) {
- rb->counts.invalid++;
- continue;
- }
- if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(s.key);
- used = storeDiskdDirMapBitTest(SD, s.swap_filen);
- /* If this URL already exists in the cache, does the swap log
- * appear to have a newer entry? Compare 'lastref' from the
- * swap log to e->lastref. */
- disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
- /* log entry is old, ignore it */
- rb->counts.clashcount++;
- continue;
- } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
- /* swapfile taken, same URL, newer, update meta */
- if (e->store_status == STORE_OK) {
- e->lastref = s.timestamp;
- e->timestamp = s.timestamp;
- e->expires = s.expires;
- e->lastmod = s.lastmod;
- e->flags = s.flags;
- e->refcount += s.refcount;
- storeDiskdDirUnrefObj(SD, e);
- } else {
- debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition");
- debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
- }
- continue;
- } else if (used) {
- /* swapfile in use, not by this URL, log entry is newer */
- /* This is sorta bad: the log entry should NOT be newer at this
- * point. If the log is dirty, the filesize check should have
- * caught this. If the log is clean, there should never be a
- * newer entry. */
- debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
- SD->index, s.swap_filen);
- /* I'm tempted to remove the swapfile here just to be safe,
- * but there is a bad race condition in the NOVM version if
- * the swapfile has recently been opened for writing, but
- * not yet opened for reading. Because we can't map
- * swapfiles back to StoreEntrys, we don't know the state
- * of the entry using that file. */
- /* We'll assume the existing entry is valid, probably because
- * were in a slow rebuild and the the swap file number got taken
- * and the validation procedure hasn't run. */
- assert(rb->flags.need_to_validate);
- rb->counts.clashcount++;
- continue;
- } else if (e && !disk_entry_newer) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (e) {
- /* key already exists, this swapfile not being used */
- /* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeDiskdDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeDiskdDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.dupcount++;
- } else {
- /* URL doesnt exist, swapfile not in use */
- /* load new */
- (void) 0;
- }
- /* update store_swap_size */
- rb->counts.objcount++;
- e = storeDiskdDirAddDiskRestore(SD, s.key,
- s.swap_filen,
- s.swap_file_sz,
- s.expires,
- s.timestamp,
- s.lastref,
- s.lastmod,
- s.refcount,
- s.flags,
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
-}
-
-static int
-storeDiskdDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size)
-{
- SwapDir *SD = rb->sd;
- diskdinfo_t *diskdinfo = SD->fsdata;
- int fd = -1;
- int used = 0;
- int dirs_opened = 0;
- debug(47, 3) ("storeDiskdDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
- rb->flags.init,
- rb->sd->index,
- rb->curlvl1,
- rb->curlvl2);
- if (rb->done)
- return -2;
- while (fd < 0 && rb->done == 0) {
- fd = -1;
- if (0 == rb->flags.init) { /* initialize, open first file */
- rb->done = 0;
- rb->curlvl1 = 0;
- rb->curlvl2 = 0;
- rb->in_dir = 0;
- rb->flags.init = 1;
- assert(Config.cacheSwap.n_configured > 0);
- }
- if (0 == rb->in_dir) { /* we need to read in a new directory */
- snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- rb->sd->path,
- rb->curlvl1, rb->curlvl2);
- if (dirs_opened)
- return -1;
- rb->td = opendir(rb->fullpath);
- dirs_opened++;
- if (rb->td == NULL) {
- debug(50, 1) ("storeDiskdDirGetNextFile: opendir: %s: %s\n",
- rb->fullpath, xstrerror());
- } else {
- rb->entry = readdir(rb->td); /* skip . and .. */
- rb->entry = readdir(rb->td);
- if (rb->entry == NULL && errno == ENOENT)
- debug(47, 1) ("storeDiskdDirGetNextFile: directory does not exist!.\n");
- debug(47, 3) ("storeDiskdDirGetNextFile: Directory %s\n", rb->fullpath);
- }
- }
- if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
- rb->in_dir++;
- if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
- debug(47, 3) ("storeDiskdDirGetNextFile: invalid %s\n",
- rb->entry->d_name);
- continue;
- }
- if (!storeDiskdFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
- debug(47, 3) ("storeDiskdDirGetNextFile: %08X does not belong in %d/%d/%d\n",
- rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
- continue;
- }
- used = storeDiskdDirMapBitTest(SD, rb->fn);
- if (used) {
- debug(47, 3) ("storeDiskdDirGetNextFile: Locked, continuing with next.\n");
- continue;
- }
- snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
- rb->fullpath, rb->entry->d_name);
- debug(47, 3) ("storeDiskdDirGetNextFile: Opening %s\n", rb->fullfilename);
- fd = file_open(rb->fullfilename, O_RDONLY);
- if (fd < 0)
- debug(50, 1) ("storeDiskdDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
- else
- store_open_disk_fd++;
- continue;
- }
- if (rb->td != NULL)
- closedir(rb->td);
- rb->td = NULL;
- rb->in_dir = 0;
- if (++rb->curlvl2 < diskdinfo->l2)
- continue;
- rb->curlvl2 = 0;
- if (++rb->curlvl1 < diskdinfo->l1)
- continue;
- rb->curlvl1 = 0;
- rb->done = 1;
- }
- *filn_p = rb->fn;
- return fd;
-}
-
-/* Add a new object to the cache with empty memory copy and pointer to disk
- * use to rebuild store from disk. */
-static StoreEntry *
-storeDiskdDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean)
-{
- StoreEntry *e = NULL;
- debug(47, 5) ("storeDiskdAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
- /* if you call this you'd better be sure file_number is not
- * already in use! */
- e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
- e->store_status = STORE_OK;
- storeSetMemStatus(e, NOT_IN_MEMORY);
- e->swap_status = SWAPOUT_DONE;
- e->swap_filen = file_number;
- e->swap_dirn = SD->index;
- e->swap_file_sz = swap_file_sz;
- e->lock_count = 0;
- e->lastref = lastref;
- e->timestamp = timestamp;
- e->expires = expires;
- e->lastmod = lastmod;
- e->refcount = refcount;
- e->flags = flags;
- EBIT_SET(e->flags, ENTRY_CACHABLE);
- EBIT_CLR(e->flags, RELEASE_REQUEST);
- EBIT_CLR(e->flags, KEY_PRIVATE);
- e->ping_status = PING_NONE;
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- storeDiskdDirMapBitSet(SD, e->swap_filen);
- storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
- storeDiskdDirReplAdd(SD, e);
- return e;
-}
-
-CBDATA_TYPE(RebuildState);
-
-static void
-storeDiskdDirRebuild(SwapDir * sd)
-{
- RebuildState *rb;
- int clean = 0;
- int zero = 0;
- FILE *fp;
- EVH *func = NULL;
- CBDATA_INIT_TYPE(RebuildState);
- rb = cbdataAlloc(RebuildState);
- rb->sd = sd;
- rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
- /*
- * If the swap.state file exists in the cache_dir, then
- * we'll use storeDiskdDirRebuildFromSwapLog(), otherwise we'll
- * use storeDiskdDirRebuildFromDirectory() to open up each file
- * and suck in the meta data.
- */
- fp = storeDiskdDirOpenTmpSwapLog(sd, &clean, &zero);
- if (fp == NULL || zero) {
- if (fp != NULL)
- fclose(fp);
- func = storeDiskdDirRebuildFromDirectory;
- } else {
- func = storeDiskdDirRebuildFromSwapLog;
- rb->log = fp;
- rb->flags.clean = (unsigned int) clean;
- }
- if (!clean)
- rb->flags.need_to_validate = 1;
- debug(47, 1) ("Rebuilding storage in %s (%s)\n",
- sd->path, clean ? "CLEAN" : "DIRTY");
- store_dirs_rebuilding++;
- eventAdd("storeRebuild", func, rb, 0.0, 1);
-}
-
-static void
-storeDiskdDirCloseTmpSwapLog(SwapDir * sd)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
- char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new"));
- int fd;
- file_close(diskdinfo->swaplog_fd);
-#ifdef _SQUID_OS2_
- if (unlink(swaplog_path) < 0) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeDiskdDirCloseTmpSwapLog: unlink failed");
- }
-#endif
- if (xrename(new_path, swaplog_path) < 0) {
- fatal("storeDiskdDirCloseTmpSwapLog: rename failed");
- }
- fd = file_open(swaplog_path, O_WRONLY | O_CREAT);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeDiskdDirCloseTmpSwapLog: Failed to open swap log.");
- }
- safe_free(swaplog_path);
- safe_free(new_path);
- diskdinfo->swaplog_fd = fd;
- debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
-}
-
-static FILE *
-storeDiskdDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
- char *clean_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean"));
- char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new"));
- struct stat log_sb;
- struct stat clean_sb;
- FILE *fp;
- int fd;
- if (stat(swaplog_path, &log_sb) < 0) {
- debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return NULL;
- }
- *zero_flag = log_sb.st_size == 0 ? 1 : 0;
- /* close the existing write-only FD */
- if (diskdinfo->swaplog_fd >= 0)
- file_close(diskdinfo->swaplog_fd);
- /* open a write-only FD for the new log */
- fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", new_path, xstrerror());
- fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
- }
- diskdinfo->swaplog_fd = fd;
- /* open a read-only stream of the old log */
- fp = fopen(swaplog_path, "rb");
- if (fp == NULL) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("Failed to open swap log for reading");
- }
- memset(&clean_sb, '\0', sizeof(struct stat));
- if (stat(clean_path, &clean_sb) < 0)
- *clean_flag = 0;
- else if (clean_sb.st_mtime < log_sb.st_mtime)
- *clean_flag = 0;
- else
- *clean_flag = 1;
- safeunlink(clean_path, 1);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return fp;
-}
-
-struct _clean_state {
- char *cur;
- char *new;
- char *cln;
- char *outbuf;
- off_t outbuf_offset;
- int fd;
- RemovalPolicyWalker *walker;
-};
-
-#define CLEAN_BUF_SZ 16384
-/*
- * Begin the process to write clean cache state. For DISKD this means
- * opening some log files and allocating write buffers. Return 0 if
- * we succeed, and assign the 'func' and 'data' return pointers.
- */
-static int
-storeDiskdDirWriteCleanStart(SwapDir * sd)
-{
- struct _clean_state *state = xcalloc(1, sizeof(*state));
-#if HAVE_FCHMOD
- struct stat sb;
-#endif
- sd->log.clean.write = NULL;
- sd->log.clean.state = NULL;
- state->new = xstrdup(storeDiskdDirSwapLogFile(sd, ".clean"));
- state->cur = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
- state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean"));
- state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
- state->outbuf_offset = 0;
- state->walker = sd->repl->WalkInit(sd->repl);
- unlink(state->cln);
- state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
- if (state->fd < 0) {
- xfree(state->new);
- xfree(state->cur);
- xfree(state->cln);
- xfree(state);
- return -1;
- }
- debug(47, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
- state->new, state->fd);
-#if HAVE_FCHMOD
- if (stat(state->cur, &sb) == 0)
- fchmod(state->fd, sb.st_mode);
-#endif
- sd->log.clean.write = storeDiskdDirWriteCleanEntry;
- sd->log.clean.state = state;
- return 0;
-}
-
-/*
- * Get the next entry that is a candidate for clean log writing
- */
-const StoreEntry *
-storeDiskdDirCleanLogNextEntry(SwapDir * sd)
-{
- const StoreEntry *entry = NULL;
- struct _clean_state *state = sd->log.clean.state;
- if (state->walker)
- entry = state->walker->Next(state->walker);
- return entry;
-}
-
-/*
- * "write" an entry to the clean log file.
- */
-static void
-storeDiskdDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
-{
- storeSwapLogData s;
- static size_t ss = sizeof(storeSwapLogData);
- struct _clean_state *state = sd->log.clean.state;
- memset(&s, '\0', ss);
- s.op = (char) SWAP_LOG_ADD;
- s.swap_filen = e->swap_filen;
- s.timestamp = e->timestamp;
- s.lastref = e->lastref;
- s.expires = e->expires;
- s.lastmod = e->lastmod;
- s.swap_file_sz = e->swap_file_sz;
- s.refcount = e->refcount;
- s.flags = e->flags;
- xmemcpy(&s.key, e->hash.key, MD5_DIGEST_CHARS);
- xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
- state->outbuf_offset += ss;
- /* buffered write */
- if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(47, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
- return;
- }
- state->outbuf_offset = 0;
- }
-}
-
-static void
-storeDiskdDirWriteCleanDone(SwapDir * sd)
-{
- struct _clean_state *state = sd->log.clean.state;
- if (NULL == state)
- return;
- if (state->fd < 0)
- return;
- state->walker->Done(state->walker);
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(47, 0) ("storeDirWriteCleanLogs: Current swap logfile "
- "not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- }
- safe_free(state->outbuf);
- /*
- * You can't rename open files on Microsoft "operating systems"
- * so we have to close before renaming.
- */
- storeDiskdDirCloseSwapLog(sd);
- /* rename */
- if (state->fd >= 0) {
-#ifdef _SQUID_OS2_
- file_close(state->fd);
- state->fd = -1;
- if (unlink(cur) < 0)
- debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
- xstrerror(), cur);
-#endif
- xrename(state->new, state->cur);
- }
- /* touch a timestamp file if we're not still validating */
- if (store_dirs_rebuilding)
- (void) 0;
- else if (state->fd < 0)
- (void) 0;
- else
- file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC));
- /* close */
- safe_free(state->cur);
- safe_free(state->new);
- safe_free(state->cln);
- if (state->fd >= 0)
- file_close(state->fd);
- state->fd = -1;
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
-}
-
-static void
-storeSwapLogDataFree(void *s)
-{
- memFree(s, MEM_SWAP_LOG_DATA);
-}
-
-static void
-storeDiskdDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
-{
- diskdinfo_t *diskdinfo = sd->fsdata;
- storeSwapLogData *s = memAllocate(MEM_SWAP_LOG_DATA);
- s->op = (char) op;
- s->swap_filen = e->swap_filen;
- s->timestamp = e->timestamp;
- s->lastref = e->lastref;
- s->expires = e->expires;
- s->lastmod = e->lastmod;
- s->swap_file_sz = e->swap_file_sz;
- s->refcount = e->refcount;
- s->flags = e->flags;
- xmemcpy(s->key, e->hash.key, MD5_DIGEST_CHARS);
- file_write(diskdinfo->swaplog_fd,
- -1,
- s,
- sizeof(storeSwapLogData),
- NULL,
- NULL,
- (FREE *) storeSwapLogDataFree);
-}
-
-static void
-storeDiskdDirNewfs(SwapDir * sd)
-{
- debug(47, 3) ("Creating swap space in %s\n", sd->path);
- storeDiskdDirCreateDirectory(sd->path, 0);
- storeDiskdDirCreateSwapSubDirs(sd);
-}
-
-static int
-rev_int_sort(const void *A, const void *B)
-{
- const int *i1 = A;
- const int *i2 = B;
- return *i2 - *i1;
-}
-
-static int
-storeDiskdDirClean(int swap_index)
-{
- DIR *dp = NULL;
- struct dirent *de = NULL;
- LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
- LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
-#if USE_TRUNCATE
- struct stat sb;
-#endif
- int files[20];
- int swapfileno;
- int fn; /* same as swapfileno, but with dirn bits set */
- int n = 0;
- int k = 0;
- int N0, N1, N2;
- int D0, D1, D2;
- SwapDir *SD;
- diskdinfo_t *diskdinfo;
- N0 = n_diskd_dirs;
- D0 = diskd_dir_index[swap_index % N0];
- SD = &Config.cacheSwap.swapDirs[D0];
- diskdinfo = SD->fsdata;
- N1 = diskdinfo->l1;
- D1 = (swap_index / N0) % N1;
- N2 = diskdinfo->l2;
- D2 = ((swap_index / N0) / N1) % N2;
- snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- Config.cacheSwap.swapDirs[D0].path, D1, D2);
- debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
- dp = opendir(p1);
- if (dp == NULL) {
- if (errno == ENOENT) {
- debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
- if (mkdir(p1, 0777) == 0)
- return 0;
- }
- debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
- safeunlink(p1, 1);
- return 0;
- }
- while ((de = readdir(dp)) != NULL && k < 20) {
- if (sscanf(de->d_name, "%X", &swapfileno) != 1)
- continue;
- fn = swapfileno; /* XXX should remove this cruft ! */
- if (storeDiskdDirValidFileno(SD, fn, 1))
- if (storeDiskdDirMapBitTest(SD, fn))
- if (storeDiskdFilenoBelongsHere(fn, D0, D1, D2))
- continue;
-#if USE_TRUNCATE
- if (!stat(de->d_name, &sb))
- if (sb.st_size == 0)
- continue;
-#endif
- files[k++] = swapfileno;
- }
- closedir(dp);
- if (k == 0)
- return 0;
- qsort(files, k, sizeof(int), rev_int_sort);
- if (k > 10)
- k = 10;
- for (n = 0; n < k; n++) {
- debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
- snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
-#if USE_TRUNCATE
- truncate(p2, 0);
-#else
- safeunlink(p2, 0);
-#endif
- statCounter.swap.files_cleaned++;
- }
- debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
- return k;
-}
-
-static void
-storeDiskdDirCleanEvent(void *unused)
-{
- static int swap_index = 0;
- int i;
- int j = 0;
- int n = 0;
- /*
- * Assert that there are DISKD cache_dirs configured, otherwise
- * we should never be called.
- */
- assert(n_diskd_dirs);
- if (NULL == diskd_dir_index) {
- SwapDir *sd;
- diskdinfo_t *diskdinfo;
- /*
- * Initialize the little array that translates DISKD cache_dir
- * number into the Config.cacheSwap.swapDirs array index.
- */
- diskd_dir_index = xcalloc(n_diskd_dirs, sizeof(*diskd_dir_index));
- for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
- sd = &Config.cacheSwap.swapDirs[i];
- if (!storeDiskdDirIs(sd))
- continue;
- diskd_dir_index[n++] = i;
- diskdinfo = sd->fsdata;
- j += (diskdinfo->l1 * diskdinfo->l2);
- }
- assert(n == n_diskd_dirs);
- /*
- * Start the storeDiskdDirClean() swap_index with a random
- * value. j equals the total number of DISKD level 2
- * swap directories
- */
- swap_index = (int) (squid_random() % j);
- }
- if (0 == store_dirs_rebuilding) {
- n = storeDiskdDirClean(swap_index);
- swap_index++;
- }
- eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL,
- 15.0 * exp(-0.25 * n), 1);
-}
-
-static int
-storeDiskdDirIs(SwapDir * sd)
-{
- if (strncmp(sd->type, "diskd", 3) == 0)
- return 1;
- return 0;
-}
-
-/*
- * Does swapfile number 'fn' belong in cachedir #F0,
- * level1 dir #F1, level2 dir #F2?
- */
-static int
-storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2)
-{
- int D1, D2;
- int L1, L2;
- int filn = fn;
- diskdinfo_t *diskdinfo;
- assert(F0 < Config.cacheSwap.n_configured);
- diskdinfo = Config.cacheSwap.swapDirs[F0].fsdata;
- L1 = diskdinfo->l1;
- L2 = diskdinfo->l2;
- D1 = ((filn / L2) / L2) % L1;
- if (F1 != D1)
- return 0;
- D2 = (filn / L2) % L2;
- if (F2 != D2)
- return 0;
- return 1;
-}
-
-int
-storeDiskdDirValidFileno(SwapDir * SD, sfileno filn, int flag)
-{
- diskdinfo_t *diskdinfo = SD->fsdata;
- if (filn < 0)
- return 0;
- /*
- * If flag is set it means out-of-range file number should
- * be considered invalid.
- */
- if (flag)
- if (filn > diskdinfo->map->max_n_files)
- return 0;
- return 1;
-}
-
-void
-storeDiskdDirMaintain(SwapDir * SD)
-{
- StoreEntry *e = NULL;
- int removed = 0;
- int max_scan;
- int max_remove;
- double f;
- RemovalPurgeWalker *walker;
- /* We can't delete objects while rebuilding swap */
- if (store_dirs_rebuilding) {
- return;
- } else {
- f = (double) (SD->cur_size - SD->low_size) / (SD->max_size - SD->low_size);
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
- max_scan = (int) (f * 400.0 + 100.0);
- max_remove = (int) (f * 70.0 + 10.0);
- /*
- * This is kinda cheap, but so we need this priority hack?
- */
- }
- debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
- walker = SD->repl->PurgeInit(SD->repl, max_scan);
- while (1) {
- if (SD->cur_size < SD->low_size)
- break;
- if (removed >= max_remove)
- break;
- e = walker->Next(walker);
- if (!e)
- break; /* no more objects */
- removed++;
- storeRelease(e);
- }
- walker->Done(walker);
- debug(47, (removed ? 2 : 3)) ("storeDiskdDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
- SD->path, removed, max_remove, f, max_scan);
-}
-
/*
* storeDiskdDirCheckObj
*
return diskdinfo->away * 1000 / diskdinfo->magic2;
}
-/*
- * storeDiskdDirRefObj
- *
- * This routine is called whenever an object is referenced, so we can
- * maintain replacement information within the storage fs.
- */
-void
-storeDiskdDirRefObj(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
- e->swap_filen);
- if (SD->repl->Referenced)
- SD->repl->Referenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeDiskdDirUnrefObj
- * This routine is called whenever the last reference to an object is
- * removed, to maintain replacement information within the storage fs.
- */
-void
-storeDiskdDirUnrefObj(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e,
- e->swap_dirn, e->swap_filen);
- if (SD->repl->Dereferenced)
- SD->repl->Dereferenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeDiskdDirUnlinkFile
- *
- * This is a *synchronous* unlink which is currently used in the rebuild
- * process. This is bad, but it'll have to stay until the dir rebuild
- * uses storeDiskdUnlink() ..
- */
void
-storeDiskdDirUnlinkFile(SwapDir * SD, sfileno f)
+storeDiskdDirIOUnlinkFile(char *path)
{
- debug(47, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f);
- /* storeDiskdDirMapBitReset(SD, f); */
#if USE_UNLINKD
- unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL));
+ unlinkdUnlink(path);
#elif USE_TRUNCATE
- truncate(storeDiskdDirFullPath(SD, f, NULL), 0);
+ truncate(path, 0);
#else
- unlink(storeDiskdDirFullPath(SD, f, NULL));
+ unlink(path);
#endif
+
}
-/*
- * Add and remove the given StoreEntry from the replacement policy in
- * use.
- */
-
-void
-storeDiskdDirReplAdd(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e,
- SD->index);
- SD->repl->Add(SD->repl, e, &e->repl);
-}
-
-
-void
-storeDiskdDirReplRemove(StoreEntry * e)
-{
- SwapDir *SD;
- if (e->swap_dirn < 0)
- return;
- SD = INDEXSD(e->swap_dirn);
- debug(47, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e,
- SD->index);
- SD->repl->Remove(SD->repl, e, &e->repl);
-}
-
-
-
/*
* SHM manipulation routines
*/
storeDiskdDirStats(SwapDir * SD, StoreEntry * sentry)
{
diskdinfo_t *diskdinfo = SD->fsdata;
- int totl_kb = 0;
- int free_kb = 0;
- int totl_in = 0;
- int free_in = 0;
- int x;
- storeAppendPrintf(sentry, "First level subdirectories: %d\n", diskdinfo->l1);
- storeAppendPrintf(sentry, "Second level subdirectories: %d\n", diskdinfo->l2);
- storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
- storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
- storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
- 100.0 * SD->cur_size / SD->max_size);
- storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
- diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files,
- percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files));
- x = storeDirGetUFSStats(SD->path, &totl_kb, &free_kb, &totl_in, &free_in);
- if (0 == x) {
- storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
- totl_kb - free_kb,
- totl_kb,
- percent(totl_kb - free_kb, totl_kb));
- storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
- totl_in - free_in,
- totl_in,
- percent(totl_in - free_in, totl_in));
- }
- storeAppendPrintf(sentry, "Flags:");
- if (SD->flags.selected)
- storeAppendPrintf(sentry, " SELECTED");
- if (SD->flags.read_only)
- storeAppendPrintf(sentry, " READ-ONLY");
- storeAppendPrintf(sentry, "\n");
+ commonUfsDirStats (SD, sentry);
storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away);
}
void
storeDiskdDirDump(StoreEntry * entry, SwapDir * s)
{
- diskdinfo_t *diskdinfo = s->fsdata;
- storeAppendPrintf(entry, " %d %d %d",
- s->max_size >> 10,
- diskdinfo->l1,
- diskdinfo->l2);
+ commonUfsDirDump (entry, s);
dump_cachedir_options(entry, options, s);
}
-/*
- * Only "free" the filesystem specific stuff here
- */
-static void
-storeDiskdDirFree(SwapDir * s)
-{
- diskdinfo_t *diskdinfo = s->fsdata;
- if (diskdinfo->swaplog_fd > -1) {
- file_close(diskdinfo->swaplog_fd);
- diskdinfo->swaplog_fd = -1;
- }
- filemapFreeMemory(diskdinfo->map);
- xfree(diskdinfo);
- s->fsdata = NULL; /* Will aid debugging... */
-
-}
-
-char *
-storeDiskdDirFullPath(SwapDir * SD, sfileno filn, char *fullpath)
-{
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- diskdinfo_t *diskdinfo = SD->fsdata;
- int L1 = diskdinfo->l1;
- int L2 = diskdinfo->l2;
- if (!fullpath)
- fullpath = fullfilename;
- fullpath[0] = '\0';
- snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
- SD->path,
- ((filn / L2) / L2) % L1,
- (filn / L2) % L2,
- filn);
- return fullpath;
-}
-
-/*
- * storeDiskdCleanupDoubleCheck
- *
- * This is called by storeCleanup() if -S was given on the command line.
- */
-static int
-storeDiskdCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
-{
- struct stat sb;
- if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
- storeDiskdDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
- }
- if (e->swap_file_sz != sb.st_size) {
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
- storeDiskdDirFullPath(sd, e->swap_filen, NULL));
- debug(47, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %ld, FILE SIZE: %ld\n",
- (long int) e->swap_file_sz, (long int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
- }
- return 0;
-}
-
/*
* storeDiskdDirParse
*
sd->index = index;
sd->path = xstrdup(path);
sd->max_size = size;
- diskdinfo->l1 = l1;
- diskdinfo->l2 = l2;
- diskdinfo->swaplog_fd = -1;
- diskdinfo->map = NULL; /* Debugging purposes */
- diskdinfo->suggest = 0;
+ diskdinfo->commondata.l1 = l1;
+ diskdinfo->commondata.l2 = l2;
+ diskdinfo->commondata.swaplog_fd = -1;
+ diskdinfo->commondata.map = NULL; /* Debugging purposes */
+ diskdinfo->commondata.suggest = 0;
+ diskdinfo->commondata.io.storeDirUnlinkFile = storeDiskdDirIOUnlinkFile;
diskdinfo->magic1 = 64;
diskdinfo->magic2 = 72;
sd->init = storeDiskdDirInit;
- sd->newfs = storeDiskdDirNewfs;
+ sd->newfs = commonUfsDirNewfs;
sd->dump = storeDiskdDirDump;
- sd->freefs = storeDiskdDirFree;
- sd->dblcheck = storeDiskdCleanupDoubleCheck;
+ sd->freefs = commonUfsDirFree;
+ sd->dblcheck = commonUfsCleanupDoubleCheck;
sd->statfs = storeDiskdDirStats;
- sd->maintainfs = storeDiskdDirMaintain;
+ sd->maintainfs = commonUfsDirMaintain;
sd->checkobj = storeDiskdDirCheckObj;
- sd->refobj = storeDiskdDirRefObj;
- sd->unrefobj = storeDiskdDirUnrefObj;
+ sd->refobj = commonUfsDirRefObj;
+ sd->unrefobj = commonUfsDirUnrefObj;
sd->callback = storeDiskdDirCallback;
sd->sync = storeDiskdDirSync;
sd->obj.create = storeDiskdCreate;
sd->obj.read = storeDiskdRead;
sd->obj.write = storeDiskdWrite;
sd->obj.unlink = storeDiskdUnlink;
- sd->log.open = storeDiskdDirOpenSwapLog;
- sd->log.close = storeDiskdDirCloseSwapLog;
- sd->log.write = storeDiskdDirSwapLog;
- sd->log.clean.start = storeDiskdDirWriteCleanStart;
- sd->log.clean.nextentry = storeDiskdDirCleanLogNextEntry;
- sd->log.clean.done = storeDiskdDirWriteCleanDone;
+ sd->log.open = commonUfsDirOpenSwapLog;
+ sd->log.close = commonUfsDirCloseSwapLog;
+ sd->log.write = commonUfsDirSwapLog;
+ sd->log.clean.start = commonUfsDirWriteCleanStart;
+ sd->log.clean.nextentry = commonUfsDirCleanLogNextEntry;
+ sd->log.clean.done = commonUfsDirWriteCleanDone;
parse_cachedir_options(sd, options, 0);
#ifndef __STORE_DISKD_H__
#define __STORE_DISKD_H__
+#include "ufscommon.h"
+
/*
* magic2 is the point at which we start blocking on msgsnd/msgrcv.
* If a queue has magic2 (or more) messages away, then we read the
*/
struct _diskdinfo_t {
- int swaplog_fd;
- int l1;
- int l2;
- fileMap *map;
- int suggest;
+ /* MUST BE FIRST */
+ squidufsinfo_t commondata;
int smsgid;
int rmsgid;
int wfd;
/* The diskd_state memory pool */
extern MemPool *diskd_state_pool;
-extern void storeDiskdDirMapBitReset(SwapDir *, sfileno);
-extern int storeDiskdDirMapBitAllocate(SwapDir *);
-extern char *storeDiskdDirFullPath(SwapDir * SD, sfileno filn, char *fullpath);
-extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno);
-extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *);
-extern void storeDiskdDirReplRemove(StoreEntry *);
extern void storeDiskdShmPut(SwapDir *, off_t);
extern void *storeDiskdShmGet(SwapDir *, off_t *);
extern void storeDiskdHandle(diomsg * M);
/*
- * $Id: store_io_diskd.cc,v 1.27 2002/08/08 20:12:46 hno Exp $
+ * $Id: store_io_diskd.cc,v 1.28 2002/10/12 09:45:57 robertc Exp $
*
* DEBUG: section 79 Squid-side DISKD I/O functions.
* AUTHOR: Duane Wessels
diskdstate->id = diskd_stats.sio_id++;
buf = storeDiskdShmGet(SD, &shm_offset);
- xstrncpy(buf, storeDiskdDirFullPath(SD, f, NULL), SHMBUF_BLKSZ);
+ xstrncpy(buf, commonUfsDirFullPath(SD, f, NULL), SHMBUF_BLKSZ);
x = storeDiskdSend(_MQD_OPEN,
SD,
diskdstate->id,
return NULL;
}
/* Allocate a number */
- f = storeDiskdDirMapBitAllocate(SD);
+ f = commonUfsDirMapBitAllocate(SD);
debug(79, 3) ("storeDiskdCreate: fileno %08X\n", f);
CBDATA_INIT_TYPE_FREECB(storeIOState, storeDiskdIOFreeEntry);
diskdstate->id = diskd_stats.sio_id++;
buf = storeDiskdShmGet(SD, &shm_offset);
- xstrncpy(buf, storeDiskdDirFullPath(SD, f, NULL), SHMBUF_BLKSZ);
+ xstrncpy(buf, commonUfsDirFullPath(SD, f, NULL), SHMBUF_BLKSZ);
x = storeDiskdSend(_MQD_OPEN,
SD,
diskdstate->id,
cbdataFree(sio);
return NULL;
}
- storeDiskdDirReplAdd(SD, e);
+ commonUfsDirReplAdd(SD, e);
diskd_stats.create.ops++;
return sio;
}
debug(79, 3) ("storeDiskdUnlink: dirno %d, fileno %08X\n", SD->index,
e->swap_filen);
- storeDiskdDirReplRemove(e);
- storeDiskdDirMapBitReset(SD, e->swap_filen);
+ commonUfsDirReplRemove(e);
+ commonUfsDirMapBitReset(SD, e->swap_filen);
if (diskdinfo->away >= diskdinfo->magic1) {
/* Damn, we need to issue a sync unlink here :( */
debug(79, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n");
- storeDiskdDirUnlinkFile(SD, e->swap_filen);
+ commonUfsDirUnlinkFile(SD, e->swap_filen);
return;
}
/* We can attempt a diskd unlink */
buf = storeDiskdShmGet(SD, &shm_offset);
- xstrncpy(buf, storeDiskdDirFullPath(SD, e->swap_filen, NULL), SHMBUF_BLKSZ);
+ xstrncpy(buf, commonUfsDirFullPath(SD, e->swap_filen, NULL), SHMBUF_BLKSZ);
x = storeDiskdSend(_MQD_UNLINK,
SD,
e->swap_filen,
/*
- * $Id: store_dir_ufs.cc,v 1.49 2002/09/01 15:16:36 hno Exp $
+ * $Id: store_dir_ufs.cc,v 1.50 2002/10/12 09:45:58 robertc Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
#include "squid.h"
#include "store_ufs.h"
+#include "ufscommon.h"
-#define DefaultLevelOneDirs 16
-#define DefaultLevelTwoDirs 256
-#define STORE_META_BUFSZ 4096
-
-typedef struct _RebuildState RebuildState;
-struct _RebuildState {
- SwapDir *sd;
- int n_read;
- FILE *log;
- int speed;
- int curlvl1;
- int curlvl2;
- struct {
- unsigned int need_to_validate:1;
- unsigned int clean:1;
- unsigned int init:1;
- } flags;
- int done;
- int in_dir;
- int fn;
- struct dirent *entry;
- DIR *td;
- char fullpath[SQUID_MAXPATHLEN];
- char fullfilename[SQUID_MAXPATHLEN];
- struct _store_rebuild_data counts;
-};
-
-static int n_ufs_dirs = 0;
-static int *ufs_dir_index = NULL;
MemPool *ufs_state_pool = NULL;
static int ufs_initialised = 0;
-static char *storeUfsDirSwapSubDir(SwapDir *, int subdirn);
-static int storeUfsDirCreateDirectory(const char *path, int);
-static int storeUfsDirVerifyCacheDirs(SwapDir *);
-static int storeUfsDirVerifyDirectory(const char *path);
-static void storeUfsDirCreateSwapSubDirs(SwapDir *);
-static char *storeUfsDirSwapLogFile(SwapDir *, const char *);
-static EVH storeUfsDirRebuildFromDirectory;
-static EVH storeUfsDirRebuildFromSwapLog;
-static int storeUfsDirGetNextFile(RebuildState *, sfileno *, int *size);
-static StoreEntry *storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean);
-static void storeUfsDirRebuild(SwapDir * sd);
-static void storeUfsDirCloseTmpSwapLog(SwapDir * sd);
-static FILE *storeUfsDirOpenTmpSwapLog(SwapDir *, int *, int *);
-static STLOGOPEN storeUfsDirOpenSwapLog;
-static STINIT storeUfsDirInit;
-static STFREE storeUfsDirFree;
-static STLOGCLEANSTART storeUfsDirWriteCleanStart;
-static STLOGCLEANNEXTENTRY storeUfsDirCleanLogNextEntry;
-static STLOGCLEANWRITE storeUfsDirWriteCleanEntry;
-static STLOGCLEANDONE storeUfsDirWriteCleanDone;
-static STLOGCLOSE storeUfsDirCloseSwapLog;
-static STLOGWRITE storeUfsDirSwapLog;
-static STNEWFS storeUfsDirNewfs;
static STDUMP storeUfsDirDump;
-static STMAINTAINFS storeUfsDirMaintain;
static STCHECKOBJ storeUfsDirCheckObj;
-static STREFOBJ storeUfsDirRefObj;
-static STUNREFOBJ storeUfsDirUnrefObj;
-static QS rev_int_sort;
-static int storeUfsDirClean(int swap_index);
-static EVH storeUfsDirCleanEvent;
-static int storeUfsDirIs(SwapDir * sd);
-static int storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2);
-static int storeUfsCleanupDoubleCheck(SwapDir *, StoreEntry *);
-static void storeUfsDirStats(SwapDir *, StoreEntry *);
-static void storeUfsDirInitBitmap(SwapDir *);
-static int storeUfsDirValidFileno(SwapDir *, sfileno, int);
+static void storeUfsDirIOUnlinkFile(char *path);
STSETUP storeFsSetup_ufs;
-/*
- * These functions were ripped straight out of the heart of store_dir.c.
- * They assume that the given filenum is on a ufs partiton, which may or
- * may not be true..
- * XXX this evilness should be tidied up at a later date!
- */
-
-static int
-storeUfsDirMapBitTest(SwapDir * SD, sfileno filn)
-{
- ufsinfo_t *ufsinfo;
- ufsinfo = (ufsinfo_t *) SD->fsdata;
- return file_map_bit_test(ufsinfo->map, filn);
-}
-
-static void
-storeUfsDirMapBitSet(SwapDir * SD, int fn)
-{
- sfileno filn = fn;
- ufsinfo_t *ufsinfo;
- ufsinfo = (ufsinfo_t *) SD->fsdata;
- file_map_bit_set(ufsinfo->map, filn);
-}
-
-void
-storeUfsDirMapBitReset(SwapDir * SD, int fn)
-{
- sfileno filn = fn;
- ufsinfo_t *ufsinfo;
- ufsinfo = (ufsinfo_t *) SD->fsdata;
- /*
- * We have to test the bit before calling file_map_bit_reset.
- * file_map_bit_reset doesn't do bounds checking. It assumes
- * filn is a valid file number, but it might not be because
- * the map is dynamic in size. Also clearing an already clear
- * bit puts the map counter of-of-whack.
- */
- if (file_map_bit_test(ufsinfo->map, filn))
- file_map_bit_reset(ufsinfo->map, filn);
-}
-
-int
-storeUfsDirMapBitAllocate(SwapDir * SD)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata;
- int fn;
- fn = file_map_allocate(ufsinfo->map, ufsinfo->suggest);
- file_map_bit_set(ufsinfo->map, fn);
- ufsinfo->suggest = fn + 1;
- return fn;
-}
-
-/*
- * Initialise the ufs bitmap
- *
- * If there already is a bitmap, and the numobjects is larger than currently
- * configured, we allocate a new bitmap and 'grow' the old one into it.
- */
-static void
-storeUfsDirInitBitmap(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
-
- if (ufsinfo->map == NULL) {
- /* First time */
- ufsinfo->map = file_map_create();
- } else if (ufsinfo->map->max_n_files) {
- /* it grew, need to expand */
- /* XXX We don't need it anymore .. */
- }
- /* else it shrunk, and we leave the old one in place */
-}
-
-static char *
-storeUfsDirSwapSubDir(SwapDir * sd, int subdirn)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
-
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- assert(0 <= subdirn && subdirn < ufsinfo->l1);
- snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
- return fullfilename;
-}
-
-static int
-storeUfsDirCreateDirectory(const char *path, int should_exist)
-{
- int created = 0;
- struct stat st;
- getCurrentTime();
- if (0 == stat(path, &st)) {
- if (S_ISDIR(st.st_mode)) {
- debug(47, should_exist ? 3 : 1) ("%s exists\n", path);
- } else {
- fatalf("Swap directory %s is not a directory.", path);
- }
- } else if (0 == mkdir(path, 0755)) {
- debug(47, should_exist ? 1 : 3) ("%s created\n", path);
- created = 1;
- } else {
- fatalf("Failed to make swap directory %s: %s",
- path, xstrerror());
- }
- return created;
-}
-
-static int
-storeUfsDirVerifyDirectory(const char *path)
-{
- struct stat sb;
- if (stat(path, &sb) < 0) {
- debug(47, 0) ("%s: %s\n", path, xstrerror());
- return -1;
- }
- if (S_ISDIR(sb.st_mode) == 0) {
- debug(47, 0) ("%s is not a directory\n", path);
- return -1;
- }
- return 0;
-}
-
-/*
- * This function is called by storeUfsDirInit(). If this returns < 0,
- * then Squid exits, complains about swap directories not
- * existing, and instructs the admin to run 'squid -z'
- */
-static int
-storeUfsDirVerifyCacheDirs(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- int j;
- const char *path = sd->path;
-
- if (storeUfsDirVerifyDirectory(path) < 0)
- return -1;
- for (j = 0; j < ufsinfo->l1; j++) {
- path = storeUfsDirSwapSubDir(sd, j);
- if (storeUfsDirVerifyDirectory(path) < 0)
- return -1;
- }
- return 0;
-}
-
-static void
-storeUfsDirCreateSwapSubDirs(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- int i, k;
- int should_exist;
- LOCAL_ARRAY(char, name, MAXPATHLEN);
- for (i = 0; i < ufsinfo->l1; i++) {
- snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
- if (storeUfsDirCreateDirectory(name, 0))
- should_exist = 0;
- else
- should_exist = 1;
- debug(47, 1) ("Making directories in %s\n", name);
- for (k = 0; k < ufsinfo->l2; k++) {
- snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
- storeUfsDirCreateDirectory(name, should_exist);
- }
- }
-}
-
-static char *
-storeUfsDirSwapLogFile(SwapDir * sd, const char *ext)
-{
- LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, digit, 32);
- char *pathtmp2;
- if (Config.Log.swap) {
- xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
- pathtmp2 = pathtmp;
- while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
- *pathtmp2 = '.';
- while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
- pathtmp[strlen(pathtmp) - 1] = '\0';
- for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
- snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
- if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
- strcat(path, ".");
- snprintf(digit, 32, "%02d", sd->index);
- strncat(path, digit, 3);
- }
- } else {
- xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
- strcat(path, "/swap.state");
- }
- if (ext)
- strncat(path, ext, 16);
- return path;
-}
-
-static void
-storeUfsDirOpenSwapLog(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- char *path;
- int fd;
- path = storeUfsDirSwapLogFile(sd, NULL);
- fd = file_open(path, O_WRONLY | O_CREAT | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", path, xstrerror());
- fatal("storeUfsDirOpenSwapLog: Failed to open swap log.");
- }
- debug(50, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
- ufsinfo->swaplog_fd = fd;
- if (0 == n_ufs_dirs)
- assert(NULL == ufs_dir_index);
- n_ufs_dirs++;
- assert(n_ufs_dirs <= Config.cacheSwap.n_configured);
-}
-
-static void
-storeUfsDirCloseSwapLog(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- if (ufsinfo->swaplog_fd < 0) /* not open */
- return;
- file_close(ufsinfo->swaplog_fd);
- debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
- sd->index, ufsinfo->swaplog_fd);
- ufsinfo->swaplog_fd = -1;
- n_ufs_dirs--;
- assert(n_ufs_dirs >= 0);
- if (0 == n_ufs_dirs)
- safe_free(ufs_dir_index);
-}
-
-static void
-storeUfsDirInit(SwapDir * sd)
-{
- static int started_clean_event = 0;
- static const char *errmsg =
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.";
- storeUfsDirInitBitmap(sd);
- if (storeUfsDirVerifyCacheDirs(sd) < 0)
- fatal(errmsg);
- storeUfsDirOpenSwapLog(sd);
- storeUfsDirRebuild(sd);
- if (!started_clean_event) {
- eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, 15.0, 1);
- started_clean_event = 1;
- }
- (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize);
-}
-
-static void
-storeUfsDirRebuildFromDirectory(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
- StoreEntry *e = NULL;
- StoreEntry tmpe;
- cache_key key[MD5_DIGEST_CHARS];
- sfileno filn = 0;
- int count;
- int size;
- struct stat sb;
- int swap_hdr_len;
- int fd = -1;
- tlv *tlv_list;
- tlv *t;
- assert(rb != NULL);
- debug(47, 3) ("storeUfsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
- for (count = 0; count < rb->speed; count++) {
- assert(fd == -1);
- fd = storeUfsDirGetNextFile(rb, &filn, &size);
- if (fd == -2) {
- debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- store_dirs_rebuilding--;
- storeUfsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- } else if (fd < 0) {
- continue;
- }
- assert(fd > -1);
- /* lets get file stats here */
- if (fstat(fd, &sb) < 0) {
- debug(47, 1) ("storeUfsDirRebuildFromDirectory: fstat(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFFF) == 0)
- debug(47, 3) (" %s %7d files opened so far.\n",
- rb->sd->path, rb->counts.scancount);
- debug(47, 9) ("file_in: fd=%d %08X\n", fd, filn);
- statCounter.syscalls.disk.reads++;
- if (FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
- debug(47, 1) ("storeUfsDirRebuildFromDirectory: read(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- continue;
- }
- file_close(fd);
- store_open_disk_fd--;
- fd = -1;
- swap_hdr_len = 0;
-#if USE_TRUNCATE
- if (sb.st_size == 0)
- continue;
-#endif
- tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
- if (tlv_list == NULL) {
- debug(47, 1) ("storeUfsDirRebuildFromDirectory: failed to get meta data\n");
- /* XXX shouldn't this be a call to storeUfsUnlink ? */
- storeUfsDirUnlinkFile(SD, filn);
- continue;
- }
- debug(47, 3) ("storeUfsDirRebuildFromDirectory: successful swap meta unpacking\n");
- memset(key, '\0', MD5_DIGEST_CHARS);
- memset(&tmpe, '\0', sizeof(StoreEntry));
- for (t = tlv_list; t; t = t->next) {
- switch (t->type) {
- case STORE_META_KEY:
- assert(t->length == MD5_DIGEST_CHARS);
- xmemcpy(key, t->value, MD5_DIGEST_CHARS);
- break;
- case STORE_META_STD:
- assert(t->length == STORE_HDR_METASIZE);
- xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
- break;
- default:
- break;
- }
- }
- storeSwapTLVFree(tlv_list);
- tlv_list = NULL;
- if (storeKeyNull(key)) {
- debug(47, 1) ("storeUfsDirRebuildFromDirectory: NULL key\n");
- storeUfsDirUnlinkFile(SD, filn);
- continue;
- }
- tmpe.hash.key = key;
- /* check sizes */
- if (tmpe.swap_file_sz == 0) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz != sb.st_size) {
- debug(47, 1) ("storeUfsDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n",
- (long int) tmpe.swap_file_sz, (long int) sb.st_size);
- storeUfsDirUnlinkFile(SD, filn);
- continue;
- }
- if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- storeUfsDirUnlinkFile(SD, filn);
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(key);
- if (e && e->lastref >= tmpe.lastref) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (NULL != e) {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- storeRelease(e); /* release old entry */
- rb->counts.dupcount++;
- }
- rb->counts.objcount++;
- storeEntryDump(&tmpe, 5);
- e = storeUfsDirAddDiskRestore(SD, key,
- filn,
- tmpe.swap_file_sz,
- tmpe.expires,
- tmpe.timestamp,
- tmpe.lastref,
- tmpe.lastmod,
- tmpe.refcount, /* refcount */
- tmpe.flags, /* flags */
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeUfsDirRebuildFromDirectory, rb, 0.0, 1);
-}
-
-static void
-storeUfsDirRebuildFromSwapLog(void *data)
-{
- RebuildState *rb = data;
- SwapDir *SD = rb->sd;
- StoreEntry *e = NULL;
- storeSwapLogData s;
- size_t ss = sizeof(storeSwapLogData);
- int count;
- int used; /* is swapfile already in use? */
- int disk_entry_newer; /* is the log entry newer than current entry? */
- double x;
- assert(rb != NULL);
- /* load a number of objects per invocation */
- for (count = 0; count < rb->speed; count++) {
- if (fread(&s, ss, 1, rb->log) != 1) {
- debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
- rb->sd->path, rb->n_read);
- fclose(rb->log);
- rb->log = NULL;
- store_dirs_rebuilding--;
- storeUfsDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
- return;
- }
- rb->n_read++;
- if (s.op <= SWAP_LOG_NOP)
- continue;
- if (s.op >= SWAP_LOG_MAX)
- continue;
- /*
- * BC: during 2.4 development, we changed the way swap file
- * numbers are assigned and stored. The high 16 bits used
- * to encode the SD index number. There used to be a call
- * to storeDirProperFileno here that re-assigned the index
- * bits. Now, for backwards compatibility, we just need
- * to mask it off.
- */
- s.swap_filen &= 0x00FFFFFF;
- debug(47, 3) ("storeUfsDirRebuildFromSwapLog: %s %s %08X\n",
- swap_log_op_str[(int) s.op],
- storeKeyText(s.key),
- s.swap_filen);
- if (s.op == SWAP_LOG_ADD) {
- (void) 0;
- } else if (s.op == SWAP_LOG_DEL) {
- if ((e = storeGet(s.key)) != NULL) {
- /*
- * Make sure we don't unlink the file, it might be
- * in use by a subsequent entry. Also note that
- * we don't have to subtract from store_swap_size
- * because adding to store_swap_size happens in
- * the cleanup procedure.
- */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.objcount--;
- rb->counts.cancelcount++;
- }
- continue;
- } else {
- x = log(++rb->counts.bad_log_op) / log(10.0);
- if (0.0 == x - (double) (int) x)
- debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
- rb->counts.bad_log_op);
- rb->counts.invalid++;
- continue;
- }
- if ((++rb->counts.scancount & 0xFFF) == 0) {
- struct stat sb;
- if (0 == fstat(fileno(rb->log), &sb))
- storeRebuildProgress(SD->index,
- (int) sb.st_size / ss, rb->n_read);
- }
- if (!storeUfsDirValidFileno(SD, s.swap_filen, 0)) {
- rb->counts.invalid++;
- continue;
- }
- if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
- rb->counts.badflags++;
- continue;
- }
- e = storeGet(s.key);
- used = storeUfsDirMapBitTest(SD, s.swap_filen);
- /* If this URL already exists in the cache, does the swap log
- * appear to have a newer entry? Compare 'lastref' from the
- * swap log to e->lastref. */
- disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
- /* log entry is old, ignore it */
- rb->counts.clashcount++;
- continue;
- } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
- /* swapfile taken, same URL, newer, update meta */
- if (e->store_status == STORE_OK) {
- e->lastref = s.timestamp;
- e->timestamp = s.timestamp;
- e->expires = s.expires;
- e->lastmod = s.lastmod;
- e->flags = s.flags;
- e->refcount += s.refcount;
- storeUfsDirUnrefObj(SD, e);
- } else {
- debug_trap("storeUfsDirRebuildFromSwapLog: bad condition");
- debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
- }
- continue;
- } else if (used) {
- /* swapfile in use, not by this URL, log entry is newer */
- /* This is sorta bad: the log entry should NOT be newer at this
- * point. If the log is dirty, the filesize check should have
- * caught this. If the log is clean, there should never be a
- * newer entry. */
- debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
- SD->index, s.swap_filen);
- /* I'm tempted to remove the swapfile here just to be safe,
- * but there is a bad race condition in the NOVM version if
- * the swapfile has recently been opened for writing, but
- * not yet opened for reading. Because we can't map
- * swapfiles back to StoreEntrys, we don't know the state
- * of the entry using that file. */
- /* We'll assume the existing entry is valid, probably because
- * were in a slow rebuild and the the swap file number got taken
- * and the validation procedure hasn't run. */
- assert(rb->flags.need_to_validate);
- rb->counts.clashcount++;
- continue;
- } else if (e && !disk_entry_newer) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- rb->counts.dupcount++;
- continue;
- } else if (e) {
- /* key already exists, this swapfile not being used */
- /* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_filen > -1) {
- storeUfsDirReplRemove(e);
- /* Make sure we don't actually unlink the file */
- storeUfsDirMapBitReset(SD, e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- }
- storeRelease(e);
- rb->counts.dupcount++;
- } else {
- /* URL doesnt exist, swapfile not in use */
- /* load new */
- (void) 0;
- }
- /* update store_swap_size */
- rb->counts.objcount++;
- e = storeUfsDirAddDiskRestore(SD, s.key,
- s.swap_filen,
- s.swap_file_sz,
- s.expires,
- s.timestamp,
- s.lastref,
- s.lastmod,
- s.refcount,
- s.flags,
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
- }
- eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
-}
-
-static int
-storeUfsDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size)
-{
- SwapDir *SD = rb->sd;
- ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata;
- int fd = -1;
- int used = 0;
- int dirs_opened = 0;
- debug(47, 3) ("storeUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
- rb->flags.init,
- rb->sd->index,
- rb->curlvl1, rb->curlvl2);
- if (rb->done)
- return -2;
- while (fd < 0 && rb->done == 0) {
- fd = -1;
- if (0 == rb->flags.init) { /* initialize, open first file */
- rb->done = 0;
- rb->curlvl1 = 0;
- rb->curlvl2 = 0;
- rb->in_dir = 0;
- rb->flags.init = 1;
- assert(Config.cacheSwap.n_configured > 0);
- }
- if (0 == rb->in_dir) { /* we need to read in a new directory */
- snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- rb->sd->path,
- rb->curlvl1,
- rb->curlvl2);
- if (dirs_opened)
- return -1;
- rb->td = opendir(rb->fullpath);
- dirs_opened++;
- if (rb->td == NULL) {
- debug(47, 1) ("storeUfsDirGetNextFile: opendir: %s: %s\n",
- rb->fullpath, xstrerror());
- } else {
- rb->entry = readdir(rb->td); /* skip . and .. */
- rb->entry = readdir(rb->td);
- if (rb->entry == NULL && errno == ENOENT)
- debug(47, 1) ("storeUfsDirGetNextFile: directory does not exist!.\n");
- debug(47, 3) ("storeUfsDirGetNextFile: Directory %s\n", rb->fullpath);
- }
- }
- if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
- rb->in_dir++;
- if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
- debug(47, 3) ("storeUfsDirGetNextFile: invalid %s\n",
- rb->entry->d_name);
- continue;
- }
- if (!storeUfsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
- debug(47, 3) ("storeUfsDirGetNextFile: %08X does not belong in %d/%d/%d\n",
- rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
- continue;
- }
- used = storeUfsDirMapBitTest(SD, rb->fn);
- if (used) {
- debug(47, 3) ("storeUfsDirGetNextFile: Locked, continuing with next.\n");
- continue;
- }
- snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
- rb->fullpath, rb->entry->d_name);
- debug(47, 3) ("storeUfsDirGetNextFile: Opening %s\n", rb->fullfilename);
- fd = file_open(rb->fullfilename, O_RDONLY | O_BINARY);
- if (fd < 0)
- debug(47, 1) ("storeUfsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
- else
- store_open_disk_fd++;
- continue;
- }
- if (rb->td != NULL)
- closedir(rb->td);
- rb->td = NULL;
- rb->in_dir = 0;
- if (++rb->curlvl2 < ufsinfo->l2)
- continue;
- rb->curlvl2 = 0;
- if (++rb->curlvl1 < ufsinfo->l1)
- continue;
- rb->curlvl1 = 0;
- rb->done = 1;
- }
- *filn_p = rb->fn;
- return fd;
-}
-
-/* Add a new object to the cache with empty memory copy and pointer to disk
- * use to rebuild store from disk. */
-static StoreEntry *
-storeUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_int32_t refcount,
- u_int16_t flags,
- int clean)
-{
- StoreEntry *e = NULL;
- debug(47, 5) ("storeUfsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
- /* if you call this you'd better be sure file_number is not
- * already in use! */
- e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
- e->store_status = STORE_OK;
- storeSetMemStatus(e, NOT_IN_MEMORY);
- e->swap_status = SWAPOUT_DONE;
- e->swap_filen = file_number;
- e->swap_dirn = SD->index;
- e->swap_file_sz = swap_file_sz;
- e->lock_count = 0;
- e->lastref = lastref;
- e->timestamp = timestamp;
- e->expires = expires;
- e->lastmod = lastmod;
- e->refcount = refcount;
- e->flags = flags;
- EBIT_SET(e->flags, ENTRY_CACHABLE);
- EBIT_CLR(e->flags, RELEASE_REQUEST);
- EBIT_CLR(e->flags, KEY_PRIVATE);
- e->ping_status = PING_NONE;
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- storeUfsDirMapBitSet(SD, e->swap_filen);
- storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
- storeUfsDirReplAdd(SD, e);
- return e;
-}
-
-CBDATA_TYPE(RebuildState);
-static void
-storeUfsDirRebuild(SwapDir * sd)
-{
- RebuildState *rb;
- int clean = 0;
- int zero = 0;
- FILE *fp;
- EVH *func = NULL;
- CBDATA_INIT_TYPE(RebuildState);
- rb = cbdataAlloc(RebuildState);
- rb->sd = sd;
- rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
- /*
- * If the swap.state file exists in the cache_dir, then
- * we'll use storeUfsDirRebuildFromSwapLog(), otherwise we'll
- * use storeUfsDirRebuildFromDirectory() to open up each file
- * and suck in the meta data.
- */
- fp = storeUfsDirOpenTmpSwapLog(sd, &clean, &zero);
- if (fp == NULL || zero) {
- if (fp != NULL)
- fclose(fp);
- func = storeUfsDirRebuildFromDirectory;
- } else {
- func = storeUfsDirRebuildFromSwapLog;
- rb->log = fp;
- rb->flags.clean = (unsigned int) clean;
- }
- if (!clean)
- rb->flags.need_to_validate = 1;
- debug(47, 1) ("Rebuilding storage in %s (%s)\n",
- sd->path, clean ? "CLEAN" : "DIRTY");
- store_dirs_rebuilding++;
- eventAdd("storeRebuild", func, rb, 0.0, 1);
-}
-
-static void
-storeUfsDirCloseTmpSwapLog(SwapDir * sd)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
- char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new"));
- int fd;
- file_close(ufsinfo->swaplog_fd);
-#if defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) || defined(_SQUID_MSWIN_)
- if (unlink(swaplog_path) < 0) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeUfsDirCloseTmpSwapLog: unlink failed");
- }
-#endif
- if (xrename(new_path, swaplog_path) < 0) {
- fatal("storeUfsDirCloseTmpSwapLog: rename failed");
- }
- fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("storeUfsDirCloseTmpSwapLog: Failed to open swap log.");
- }
- safe_free(swaplog_path);
- safe_free(new_path);
- ufsinfo->swaplog_fd = fd;
- debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
-}
-
-static FILE *
-storeUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
- char *clean_path = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
- char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new"));
- struct stat log_sb;
- struct stat clean_sb;
- FILE *fp;
- int fd;
- if (stat(swaplog_path, &log_sb) < 0) {
- debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return NULL;
- }
- *zero_flag = log_sb.st_size == 0 ? 1 : 0;
- /* close the existing write-only FD */
- if (ufsinfo->swaplog_fd >= 0)
- file_close(ufsinfo->swaplog_fd);
- /* open a write-only FD for the new log */
- fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
- if (fd < 0) {
- debug(50, 1) ("%s: %s\n", new_path, xstrerror());
- fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
- }
- ufsinfo->swaplog_fd = fd;
- /* open a read-only stream of the old log */
- fp = fopen(swaplog_path, "rb");
- if (fp == NULL) {
- debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
- fatal("Failed to open swap log for reading");
- }
- memset(&clean_sb, '\0', sizeof(struct stat));
- if (stat(clean_path, &clean_sb) < 0)
- *clean_flag = 0;
- else if (clean_sb.st_mtime < log_sb.st_mtime)
- *clean_flag = 0;
- else
- *clean_flag = 1;
- safeunlink(clean_path, 1);
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
- return fp;
-}
-
-struct _clean_state {
- char *cur;
- char *new;
- char *cln;
- char *outbuf;
- off_t outbuf_offset;
- int fd;
- RemovalPolicyWalker *walker;
-};
-
-#define CLEAN_BUF_SZ 16384
-/*
- * Begin the process to write clean cache state. For UFS this means
- * opening some log files and allocating write buffers. Return 0 if
- * we succeed, and assign the 'func' and 'data' return pointers.
- */
-static int
-storeUfsDirWriteCleanStart(SwapDir * sd)
-{
- struct _clean_state *state = xcalloc(1, sizeof(*state));
-#if HAVE_FCHMOD
- struct stat sb;
-#endif
- sd->log.clean.write = NULL;
- sd->log.clean.state = NULL;
- state->new = xstrdup(storeUfsDirSwapLogFile(sd, ".clean"));
- state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
- if (state->fd < 0) {
- xfree(state->new);
- xfree(state);
- return -1;
- }
- state->cur = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
- state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
- state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
- state->outbuf_offset = 0;
- state->walker = sd->repl->WalkInit(sd->repl);
- unlink(state->cln);
- debug(47, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
- state->new, state->fd);
-#if HAVE_FCHMOD
- if (stat(state->cur, &sb) == 0)
- fchmod(state->fd, sb.st_mode);
-#endif
- sd->log.clean.write = storeUfsDirWriteCleanEntry;
- sd->log.clean.state = state;
- return 0;
-}
-
-/*
- * Get the next entry that is a candidate for clean log writing
- */
-const StoreEntry *
-storeUfsDirCleanLogNextEntry(SwapDir * sd)
-{
- const StoreEntry *entry = NULL;
- struct _clean_state *state = sd->log.clean.state;
- if (state->walker)
- entry = state->walker->Next(state->walker);
- return entry;
-}
-
-/*
- * "write" an entry to the clean log file.
- */
-static void
-storeUfsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
-{
- storeSwapLogData s;
- static size_t ss = sizeof(storeSwapLogData);
- struct _clean_state *state = sd->log.clean.state;
- memset(&s, '\0', ss);
- s.op = (char) SWAP_LOG_ADD;
- s.swap_filen = e->swap_filen;
- s.timestamp = e->timestamp;
- s.lastref = e->lastref;
- s.expires = e->expires;
- s.lastmod = e->lastmod;
- s.swap_file_sz = e->swap_file_sz;
- s.refcount = e->refcount;
- s.flags = e->flags;
- xmemcpy(&s.key, e->hash.key, MD5_DIGEST_CHARS);
- xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
- state->outbuf_offset += ss;
- /* buffered write */
- if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
- return;
- }
- state->outbuf_offset = 0;
- }
-}
-
-static void
-storeUfsDirWriteCleanDone(SwapDir * sd)
-{
- int fd;
- struct _clean_state *state = sd->log.clean.state;
- if (NULL == state)
- return;
- if (state->fd < 0)
- return;
- state->walker->Done(state->walker);
- if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
- state->new, xstrerror());
- debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile "
- "not replaced.\n");
- file_close(state->fd);
- state->fd = -1;
- unlink(state->new);
- }
- safe_free(state->outbuf);
- /*
- * You can't rename open files on Microsoft "operating systems"
- * so we have to close before renaming.
- */
- storeUfsDirCloseSwapLog(sd);
- /* save the fd value for a later test */
- fd = state->fd;
- /* rename */
- if (state->fd >= 0) {
-#if defined(_SQUID_OS2_) || defined(_SQUID_CYGWIN_) || defined(_SQUID_MSWIN_)
- file_close(state->fd);
- state->fd = -1;
- if (unlink(state->cur) < 0)
- debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
- xstrerror(), state->cur);
-#endif
- xrename(state->new, state->cur);
- }
- /* touch a timestamp file if we're not still validating */
- if (store_dirs_rebuilding)
- (void) 0;
- else if (fd < 0)
- (void) 0;
- else
- file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
- /* close */
- safe_free(state->cur);
- safe_free(state->new);
- safe_free(state->cln);
- if (state->fd >= 0)
- file_close(state->fd);
- state->fd = -1;
- safe_free(state);
- sd->log.clean.state = NULL;
- sd->log.clean.write = NULL;
-}
-
-static void
-storeSwapLogDataFree(void *s)
-{
- memFree(s, MEM_SWAP_LOG_DATA);
-}
-
-static void
-storeUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) sd->fsdata;
- storeSwapLogData *s = memAllocate(MEM_SWAP_LOG_DATA);
- s->op = (char) op;
- s->swap_filen = e->swap_filen;
- s->timestamp = e->timestamp;
- s->lastref = e->lastref;
- s->expires = e->expires;
- s->lastmod = e->lastmod;
- s->swap_file_sz = e->swap_file_sz;
- s->refcount = e->refcount;
- s->flags = e->flags;
- xmemcpy(s->key, e->hash.key, MD5_DIGEST_CHARS);
- file_write(ufsinfo->swaplog_fd,
- -1,
- s,
- sizeof(storeSwapLogData),
- NULL,
- NULL,
- (FREE *) storeSwapLogDataFree);
-}
-
-static void
-storeUfsDirNewfs(SwapDir * sd)
-{
- debug(47, 3) ("Creating swap space in %s\n", sd->path);
- storeUfsDirCreateDirectory(sd->path, 0);
- storeUfsDirCreateSwapSubDirs(sd);
-}
-
-static int
-rev_int_sort(const void *A, const void *B)
-{
- const int *i1 = A;
- const int *i2 = B;
- return *i2 - *i1;
-}
-
-static int
-storeUfsDirClean(int swap_index)
-{
- DIR *dp = NULL;
- struct dirent *de = NULL;
- LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
- LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
-#if USE_TRUNCATE
- struct stat sb;
-#endif
- int files[20];
- int swapfileno;
- int fn; /* same as swapfileno, but with dirn bits set */
- int n = 0;
- int k = 0;
- int N0, N1, N2;
- int D0, D1, D2;
- SwapDir *SD;
- ufsinfo_t *ufsinfo;
- N0 = n_ufs_dirs;
- D0 = ufs_dir_index[swap_index % N0];
- SD = &Config.cacheSwap.swapDirs[D0];
- ufsinfo = (ufsinfo_t *) SD->fsdata;
- N1 = ufsinfo->l1;
- D1 = (swap_index / N0) % N1;
- N2 = ufsinfo->l2;
- D2 = ((swap_index / N0) / N1) % N2;
- snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- Config.cacheSwap.swapDirs[D0].path, D1, D2);
- debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
- dp = opendir(p1);
- if (dp == NULL) {
- if (errno == ENOENT) {
- debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
- if (mkdir(p1, 0777) == 0)
- return 0;
- }
- debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
- safeunlink(p1, 1);
- return 0;
- }
- while ((de = readdir(dp)) != NULL && k < 20) {
- if (sscanf(de->d_name, "%X", &swapfileno) != 1)
- continue;
- fn = swapfileno; /* XXX should remove this cruft ! */
- if (storeUfsDirValidFileno(SD, fn, 1))
- if (storeUfsDirMapBitTest(SD, fn))
- if (storeUfsFilenoBelongsHere(fn, D0, D1, D2))
- continue;
-#if USE_TRUNCATE
- if (!stat(de->d_name, &sb))
- if (sb.st_size == 0)
- continue;
-#endif
- files[k++] = swapfileno;
- }
- closedir(dp);
- if (k == 0)
- return 0;
- qsort(files, k, sizeof(int), rev_int_sort);
- if (k > 10)
- k = 10;
- for (n = 0; n < k; n++) {
- debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
- snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
-#if USE_TRUNCATE
- truncate(p2, 0);
-#else
- safeunlink(p2, 0);
-#endif
- statCounter.swap.files_cleaned++;
- }
- debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
- return k;
-}
-
-static void
-storeUfsDirCleanEvent(void *unused)
-{
- static int swap_index = 0;
- int i;
- int j = 0;
- int n = 0;
- /*
- * Assert that there are UFS cache_dirs configured, otherwise
- * we should never be called.
- */
- assert(n_ufs_dirs);
- if (NULL == ufs_dir_index) {
- SwapDir *sd;
- ufsinfo_t *ufsinfo;
- /*
- * Initialize the little array that translates UFS cache_dir
- * number into the Config.cacheSwap.swapDirs array index.
- */
- ufs_dir_index = xcalloc(n_ufs_dirs, sizeof(*ufs_dir_index));
- for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
- sd = &Config.cacheSwap.swapDirs[i];
- if (!storeUfsDirIs(sd))
- continue;
- ufs_dir_index[n++] = i;
- ufsinfo = (ufsinfo_t *) sd->fsdata;
- j += (ufsinfo->l1 * ufsinfo->l2);
- }
- assert(n == n_ufs_dirs);
- /*
- * Start the storeUfsDirClean() swap_index with a random
- * value. j equals the total number of UFS level 2
- * swap directories
- */
- swap_index = (int) (squid_random() % j);
- }
- if (0 == store_dirs_rebuilding) {
- n = storeUfsDirClean(swap_index);
- swap_index++;
- }
- eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL,
- 15.0 * exp(-0.25 * n), 1);
-}
-
-static int
-storeUfsDirIs(SwapDir * sd)
-{
- if (strncmp(sd->type, "ufs", 3) == 0)
- return 1;
- return 0;
-}
-
-/*
- * Does swapfile number 'fn' belong in cachedir #F0,
- * level1 dir #F1, level2 dir #F2?
- */
-static int
-storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2)
-{
- int D1, D2;
- int L1, L2;
- int filn = fn;
- ufsinfo_t *ufsinfo;
- assert(F0 < Config.cacheSwap.n_configured);
- ufsinfo = (ufsinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata;
- L1 = ufsinfo->l1;
- L2 = ufsinfo->l2;
- D1 = ((filn / L2) / L2) % L1;
- if (F1 != D1)
- return 0;
- D2 = (filn / L2) % L2;
- if (F2 != D2)
- return 0;
- return 1;
-}
-
-int
-storeUfsDirValidFileno(SwapDir * SD, sfileno filn, int flag)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata;
- if (filn < 0)
- return 0;
- /*
- * If flag is set it means out-of-range file number should
- * be considered invalid.
- */
- if (flag)
- if (filn > ufsinfo->map->max_n_files)
- return 0;
- return 1;
-}
-
-void
-storeUfsDirMaintain(SwapDir * SD)
-{
- StoreEntry *e = NULL;
- int removed = 0;
- int max_scan;
- int max_remove;
- double f;
- RemovalPurgeWalker *walker;
- /* We can't delete objects while rebuilding swap */
- if (store_dirs_rebuilding) {
- return;
- } else {
- f = (double) (SD->cur_size - SD->low_size) / (SD->max_size - SD->low_size);
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
- max_scan = (int) (f * 400.0 + 100.0);
- max_remove = (int) (f * 70.0 + 10.0);
- /*
- * This is kinda cheap, but so we need this priority hack?
- */
- }
- debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
- walker = SD->repl->PurgeInit(SD->repl, max_scan);
- while (1) {
- if (SD->cur_size < SD->low_size)
- break;
- if (removed >= max_remove)
- break;
- e = walker->Next(walker);
- if (!e)
- break; /* no more objects */
- removed++;
- storeRelease(e);
- }
- walker->Done(walker);
- debug(47, (removed ? 2 : 3)) ("storeUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
- SD->path, removed, max_remove, f, max_scan);
-}
-
/*
* storeUfsDirCheckObj
*
return 999;
}
-/*
- * storeUfsDirRefObj
- *
- * This routine is called whenever an object is referenced, so we can
- * maintain replacement information within the storage fs.
- */
-void
-storeUfsDirRefObj(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
- e->swap_filen);
- if (SD->repl->Referenced)
- SD->repl->Referenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeUfsDirUnrefObj
- * This routine is called whenever the last reference to an object is
- * removed, to maintain replacement information within the storage fs.
- */
void
-storeUfsDirUnrefObj(SwapDir * SD, StoreEntry * e)
+storeUfsDirIOUnlinkFile(char *path)
{
- debug(47, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
- e->swap_filen);
- if (SD->repl->Dereferenced)
- SD->repl->Dereferenced(SD->repl, e, &e->repl);
-}
-
-/*
- * storeUfsDirUnlinkFile
- *
- * This routine unlinks a file and pulls it out of the bitmap.
- * It used to be in storeUfsUnlink(), however an interface change
- * forced this bit of code here. Eeek.
- */
-void
-storeUfsDirUnlinkFile(SwapDir * SD, sfileno f)
-{
- debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f);
- /* storeUfsDirMapBitReset(SD, f); */
#if USE_UNLINKD
- unlinkdUnlink(storeUfsDirFullPath(SD, f, NULL));
+ unlinkdUnlink(path);
#elif USE_TRUNCATE
- truncate(storeUfsDirFullPath(SD, f, NULL), 0);
+ truncate(path, 0);
#else
- unlink(storeUfsDirFullPath(SD, f, NULL));
+ unlink(path);
#endif
}
-/*
- * Add and remove the given StoreEntry from the replacement policy in
- * use.
- */
-
-void
-storeUfsDirReplAdd(SwapDir * SD, StoreEntry * e)
-{
- debug(47, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e,
- SD->index);
- SD->repl->Add(SD->repl, e, &e->repl);
-}
-
-
-void
-storeUfsDirReplRemove(StoreEntry * e)
-{
- SwapDir *SD = INDEXSD(e->swap_dirn);
- debug(47, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e,
- SD->index);
- SD->repl->Remove(SD->repl, e, &e->repl);
-}
-
-
-
/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
-void
-storeUfsDirStats(SwapDir * SD, StoreEntry * sentry)
-{
- ufsinfo_t *ufsinfo = SD->fsdata;
- int totl_kb = 0;
- int free_kb = 0;
- int totl_in = 0;
- int free_in = 0;
- int x;
- storeAppendPrintf(sentry, "First level subdirectories: %d\n", ufsinfo->l1);
- storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ufsinfo->l2);
- storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
- storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
- storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
- 100.0 * SD->cur_size / SD->max_size);
- storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
- ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files,
- percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files));
- x = storeDirGetUFSStats(SD->path, &totl_kb, &free_kb, &totl_in, &free_in);
- if (0 == x) {
- storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
- totl_kb - free_kb,
- totl_kb,
- percent(totl_kb - free_kb, totl_kb));
- storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
- totl_in - free_in,
- totl_in,
- percent(totl_in - free_in, totl_in));
- }
- storeAppendPrintf(sentry, "Flags:");
- if (SD->flags.selected)
- storeAppendPrintf(sentry, " SELECTED");
- if (SD->flags.read_only)
- storeAppendPrintf(sentry, " READ-ONLY");
- storeAppendPrintf(sentry, "\n");
-}
-
static struct cache_dir_option options[] =
{
#if NOT_YET_DONE
void
storeUfsDirDump(StoreEntry * entry, SwapDir * s)
{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) s->fsdata;
- storeAppendPrintf(entry, " %d %d %d",
- s->max_size >> 10,
- ufsinfo->l1,
- ufsinfo->l2);
+ commonUfsDirDump (entry, s);
dump_cachedir_options(entry, options, s);
}
-/*
- * Only "free" the filesystem specific stuff here
- */
-static void
-storeUfsDirFree(SwapDir * s)
-{
- ufsinfo_t *ufsinfo = (ufsinfo_t *) s->fsdata;
- if (ufsinfo->swaplog_fd > -1) {
- file_close(ufsinfo->swaplog_fd);
- ufsinfo->swaplog_fd = -1;
- }
- filemapFreeMemory(ufsinfo->map);
- xfree(ufsinfo);
- s->fsdata = NULL; /* Will aid debugging... */
-
-}
-
-char *
-storeUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath)
-{
- LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
- ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata;
- int L1 = ufsinfo->l1;
- int L2 = ufsinfo->l2;
- if (!fullpath)
- fullpath = fullfilename;
- fullpath[0] = '\0';
- snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
- SD->path,
- ((filn / L2) / L2) % L1,
- (filn / L2) % L2,
- filn);
- return fullpath;
-}
-
-/*
- * storeUfsCleanupDoubleCheck
- *
- * This is called by storeCleanup() if -S was given on the command line.
- */
-static int
-storeUfsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
-{
- struct stat sb;
- if (stat(storeUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(47, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(47, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
- storeUfsDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
- }
- if (e->swap_file_sz != sb.st_size) {
- debug(47, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(47, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(47, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
- storeUfsDirFullPath(sd, e->swap_filen, NULL));
- debug(47, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %ld, FILE SIZE: %ld\n",
- (long int) e->swap_file_sz, (long int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
- }
- return 0;
-}
-
/*
* storeUfsDirParse
*
int size;
int l1;
int l2;
- ufsinfo_t *ufsinfo;
+ squidufsinfo_t *ufsinfo;
i = GetInteger();
size = i << 10; /* Mbytes to kbytes */
if (l2 <= 0)
fatal("storeUfsDirParse: invalid level 2 directories value");
- ufsinfo = xmalloc(sizeof(ufsinfo_t));
+ ufsinfo = xmalloc(sizeof(squidufsinfo_t));
if (ufsinfo == NULL)
- fatal("storeUfsDirParse: couldn't xmalloc() ufsinfo_t!\n");
+ fatal("storeUfsDirParse: couldn't xmalloc() squidufsinfo_t!\n");
sd->index = index;
sd->path = xstrdup(path);
ufsinfo->swaplog_fd = -1;
ufsinfo->map = NULL; /* Debugging purposes */
ufsinfo->suggest = 0;
- sd->init = storeUfsDirInit;
- sd->newfs = storeUfsDirNewfs;
+ ufsinfo->io.storeDirUnlinkFile = storeUfsDirIOUnlinkFile;
+ sd->init = commonUfsDirInit;
+ sd->newfs = commonUfsDirNewfs;
sd->dump = storeUfsDirDump;
- sd->freefs = storeUfsDirFree;
- sd->dblcheck = storeUfsCleanupDoubleCheck;
- sd->statfs = storeUfsDirStats;
- sd->maintainfs = storeUfsDirMaintain;
+ sd->freefs = commonUfsDirFree;
+ sd->dblcheck = commonUfsCleanupDoubleCheck;
+ sd->statfs = commonUfsDirStats;
+ sd->maintainfs = commonUfsDirMaintain;
sd->checkobj = storeUfsDirCheckObj;
- sd->refobj = storeUfsDirRefObj;
- sd->unrefobj = storeUfsDirUnrefObj;
+ sd->refobj = commonUfsDirRefObj;
+ sd->unrefobj = commonUfsDirUnrefObj;
sd->callback = NULL;
sd->sync = NULL;
sd->obj.create = storeUfsCreate;
sd->obj.read = storeUfsRead;
sd->obj.write = storeUfsWrite;
sd->obj.unlink = storeUfsUnlink;
- sd->log.open = storeUfsDirOpenSwapLog;
- sd->log.close = storeUfsDirCloseSwapLog;
- sd->log.write = storeUfsDirSwapLog;
- sd->log.clean.start = storeUfsDirWriteCleanStart;
- sd->log.clean.nextentry = storeUfsDirCleanLogNextEntry;
- sd->log.clean.done = storeUfsDirWriteCleanDone;
+ sd->log.open = commonUfsDirOpenSwapLog;
+ sd->log.close = commonUfsDirCloseSwapLog;
+ sd->log.write = commonUfsDirSwapLog;
+ sd->log.clean.start = commonUfsDirWriteCleanStart;
+ sd->log.clean.nextentry = commonUfsDirCleanLogNextEntry;
+ sd->log.clean.done = commonUfsDirWriteCleanDone;
parse_cachedir_options(sd, options, 1);
/*
- * $Id: store_io_ufs.cc,v 1.11 2002/08/08 20:12:46 hno Exp $
+ * $Id: store_io_ufs.cc,v 1.12 2002/10/12 09:45:58 robertc Exp $
*
* DEBUG: section 79 Storage Manager UFS Interface
* AUTHOR: Duane Wessels
#include "squid.h"
#include "store_ufs.h"
+#include "ufscommon.h"
static DRCB storeUfsReadDone;
STIOCB * callback, void *callback_data)
{
sfileno f = e->swap_filen;
- char *path = storeUfsDirFullPath(SD, f, NULL);
+ char *path = commonUfsDirFullPath(SD, f, NULL);
storeIOState *sio;
struct stat sb;
int fd;
int fd;
int mode = (O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
char *path;
- ufsinfo_t *ufsinfo = (ufsinfo_t *) SD->fsdata;
+ squidufsinfo_t *ufsinfo = (squidufsinfo_t *) SD->fsdata;
sfileno filn;
sdirno dirn;
/* Allocate a number */
dirn = SD->index;
- filn = storeUfsDirMapBitAllocate(SD);
+ filn = commonUfsDirMapBitAllocate(SD);
ufsinfo->suggest = filn + 1;
/* Shouldn't we handle a 'bitmap full' error here? */
- path = storeUfsDirFullPath(SD, filn, NULL);
+ path = commonUfsDirFullPath(SD, filn, NULL);
debug(79, 3) ("storeUfsCreate: fileno %08X\n", filn);
fd = file_open(path, mode);
store_open_disk_fd++;
/* now insert into the replacement policy */
- storeUfsDirReplAdd(SD, e);
+ commonUfsDirReplAdd(SD, e);
return sio;
}
storeUfsUnlink(SwapDir * SD, StoreEntry * e)
{
debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen);
- storeUfsDirReplRemove(e);
- storeUfsDirMapBitReset(SD, e->swap_filen);
- storeUfsDirUnlinkFile(SD, e->swap_filen);
+ commonUfsDirReplRemove(e);
+ commonUfsDirMapBitReset(SD, e->swap_filen);
+ commonUfsDirUnlinkFile(SD, e->swap_filen);
}
/* === STATIC =========================================================== */
#ifndef __STORE_UFS_H__
#define __STORE_UFS_H__
-struct _ufsinfo_t {
- int swaplog_fd;
- int l1;
- int l2;
- fileMap *map;
- int suggest;
-};
-
struct _ufsstate_t {
int fd;
struct {
} flags;
};
-typedef struct _ufsinfo_t ufsinfo_t;
typedef struct _ufsstate_t ufsstate_t;
/* The ufs_state memory pool */
extern MemPool *ufs_state_pool;
-extern void storeUfsDirMapBitReset(SwapDir *, sfileno);
-extern int storeUfsDirMapBitAllocate(SwapDir *);
-extern char *storeUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath);
-extern void storeUfsDirUnlinkFile(SwapDir *, sfileno);
-extern void storeUfsDirReplAdd(SwapDir * SD, StoreEntry *);
-extern void storeUfsDirReplRemove(StoreEntry *);
-
/*
* Store IO stuff
*/
/*
- * $Id: leakfinder.cc,v 1.5 2001/01/12 00:37:19 wessels Exp $
+ * $Id: leakfinder.cc,v 1.6 2002/10/12 09:45:56 robertc Exp $
*
* DEBUG: section 45 Callback Data Registry
* AUTHOR: Duane Wessels
while ((hptr = hash_next(htable))) {
c = (ptr *) hptr;
storeAppendPrintf(sentry, "%20p last used %9d seconds ago by %s:%d\n",
- c->key, squid_curtime - c->when, c->file, c->line);
+ c->key, (int) (squid_curtime - c->when), c->file, c->line);
}
}
--- /dev/null
+/*
+ * $Id: ufscommon.cc,v 1.1 2002/10/12 09:45:56 robertc Exp $
+ *
+ * DEBUG: section 47 Store Directory Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Web Proxy Cache http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from
+ * the Internet community; see the CONTRIBUTORS file for full
+ * details. Many organizations have provided support for Squid's
+ * development; see the SPONSORS file for full details. Squid is
+ * Copyrighted (C) 2001 by the Regents of the University of
+ * California; see the COPYRIGHT file for full details. Squid
+ * incorporates software developed and/or copyrighted by other
+ * sources; see the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "ufscommon.h"
+#if 0
+
+#include "squid.h"
+
+#include "store_asyncufs.h"
+
+#define DefaultLevelOneDirs 16
+#define DefaultLevelTwoDirs 256
+#define STORE_META_BUFSZ 4096
+
+#endif
+typedef struct _RebuildState RebuildState;
+struct _RebuildState {
+ SwapDir *sd;
+ int n_read;
+ FILE *log;
+ int speed;
+ int curlvl1;
+ int curlvl2;
+ struct {
+ unsigned int need_to_validate:1;
+ unsigned int clean:1;
+ unsigned int init:1;
+ } flags;
+ int done;
+ int in_dir;
+ int fn;
+ struct dirent *entry;
+ DIR *td;
+ char fullpath[SQUID_MAXPATHLEN];
+ char fullfilename[SQUID_MAXPATHLEN];
+ struct _store_rebuild_data counts;
+};
+
+static int n_dirs = 0;
+static int *dir_index = NULL;
+#if 0
+MemPool *squidaio_state_pool = NULL;
+MemPool *aufs_qread_pool = NULL;
+MemPool *aufs_qwrite_pool = NULL;
+static int asyncufs_initialised = 0;
+#endif
+static int commonUfsFilenoBelongsHere(int fn, int F0, int F1, int F2);
+static char *commonUfsDirSwapSubDir(SwapDir *, int subdirn);
+static int commonUfsDirCreateDirectory(const char *path, int);
+static int commonUfsDirVerifyCacheDirs(SwapDir * sd);
+static int commonUfsDirVerifyDirectory(const char *path);
+static void commonUfsDirCreateSwapSubDirs(SwapDir *);
+static int commonUfsDirMapBitTest(SwapDir * SD, sfileno filn);
+static char *commonUfsDirSwapLogFile(SwapDir *, const char *);
+static EVH commonUfsDirRebuildFromDirectory;
+static EVH commonUfsDirRebuildFromSwapLog;
+static int commonUfsDirGetNextFile(RebuildState *, sfileno *, int *size);
+static StoreEntry *commonUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
+ sfileno file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_int32_t refcount,
+ u_int16_t flags,
+ int clean);
+static void commonUfsDirRebuild(SwapDir * sd);
+static void commonUfsDirCloseTmpSwapLog(SwapDir * sd);
+static FILE *commonUfsDirOpenTmpSwapLog(SwapDir *, int *, int *);
+#if 0
+static STLOGOPEN commonUfsDirOpenSwapLog;
+static STINIT commonUfsDirInit;
+static STFREE commonUfsDirFree;
+static STLOGCLEANSTART commonUfsDirWriteCleanStart;
+static STLOGCLEANNEXTENTRY commonUfsDirCleanLogNextEntry;
+#endif
+static STLOGCLEANWRITE commonUfsDirWriteCleanEntry;
+#if 0
+static STLOGCLEANDONE commonUfsDirWriteCleanDone;
+static STLOGCLOSE commonUfsDirCloseSwapLog;
+static STLOGWRITE commonUfsDirSwapLog;
+static STNEWFS commonUfsDirNewfs;
+static STCHECKOBJ commonUfsDirCheckObj;
+#endif
+static QS rev_int_sort;
+static void commonUfsDirMapBitSet(SwapDir * SD, sfileno filn);
+static EVH commonUfsDirCleanEvent;
+static int commonUfsDirClean(int swap_index);
+static int commonUfsDirIs(SwapDir * sd);
+#if 0
+static int commonUfsCleanupDoubleCheck(SwapDir *, StoreEntry *);
+#endif
+static void commonUfsDirInitBitmap(SwapDir *);
+static int commonUfsDirValidFileno(SwapDir * SD, sfileno filn, int flag);
+static int commonUfsDirMapBitTest(SwapDir * SD, sfileno filn);
+void commonUfsDirMapBitReset(SwapDir * SD, sfileno filn);
+
+#if 0
+
+/* The MAIN externally visible function */
+STSETUP storeFsSetup_aufs;
+
+/*
+ * These functions were ripped straight out of the heart of store_dir.c.
+ * They assume that the given filenum is on a asyncufs partiton, which may or
+ * may not be true..
+ * XXX this evilness should be tidied up at a later date!
+ */
+
+#endif
+int
+commonUfsDirMapBitTest(SwapDir * SD, sfileno filn)
+{
+ squidufsinfo_t *ioinfo;
+ ioinfo = (squidufsinfo_t *) SD->fsdata;
+ return file_map_bit_test(ioinfo->map, filn);
+}
+
+void
+commonUfsDirMapBitSet(SwapDir * SD, sfileno filn)
+{
+ squidufsinfo_t *ioinfo;
+ ioinfo = (squidufsinfo_t *) SD->fsdata;
+ file_map_bit_set(ioinfo->map, filn);
+}
+
+void
+commonUfsDirMapBitReset(SwapDir * SD, sfileno filn)
+{
+ squidufsinfo_t *ioinfo;
+ ioinfo = (squidufsinfo_t *) SD->fsdata;
+ /*
+ * We have to test the bit before calling file_map_bit_reset.
+ * file_map_bit_reset doesn't do bounds checking. It assumes
+ * filn is a valid file number, but it might not be because
+ * the map is dynamic in size. Also clearing an already clear
+ * bit puts the map counter of-of-whack.
+ */
+ if (file_map_bit_test(ioinfo->map, filn))
+ file_map_bit_reset(ioinfo->map, filn);
+}
+
+int
+commonUfsDirMapBitAllocate(SwapDir * SD)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) SD->fsdata;
+ int fn;
+ fn = file_map_allocate(ioinfo->map, ioinfo->suggest);
+ file_map_bit_set(ioinfo->map, fn);
+ ioinfo->suggest = fn + 1;
+ return fn;
+}
+
+/*
+ * Initialise the asyncufs bitmap
+ *
+ * If there already is a bitmap, and the numobjects is larger than currently
+ * configured, we allocate a new bitmap and 'grow' the old one into it.
+ */
+void
+commonUfsDirInitBitmap(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+
+ if (ioinfo->map == NULL) {
+ /* First time */
+ ioinfo->map = file_map_create();
+ } else if (ioinfo->map->max_n_files) {
+ /* it grew, need to expand */
+ /* XXX We don't need it anymore .. */
+ }
+ /* else it shrunk, and we leave the old one in place */
+}
+
+char *
+commonUfsDirSwapSubDir(SwapDir * sd, int subdirn)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ assert(0 <= subdirn && subdirn < ioinfo->l1);
+ snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
+ return fullfilename;
+}
+
+int
+commonUfsDirCreateDirectory(const char *path, int should_exist)
+{
+ int created = 0;
+ struct stat st;
+ getCurrentTime();
+ if (0 == stat(path, &st)) {
+ if (S_ISDIR(st.st_mode)) {
+ debug(47, should_exist ? 3 : 1) ("%s exists\n", path);
+ } else {
+ fatalf("Swap directory %s is not a directory.", path);
+ }
+ } else if (0 == mkdir(path, 0755)) {
+ debug(47, should_exist ? 1 : 3) ("%s created\n", path);
+ created = 1;
+ } else {
+ fatalf("Failed to make swap directory %s: %s",
+ path, xstrerror());
+ }
+ return created;
+}
+
+int
+commonUfsDirVerifyDirectory(const char *path)
+{
+ struct stat sb;
+ if (stat(path, &sb) < 0) {
+ debug(47, 0) ("%s: %s\n", path, xstrerror());
+ return -1;
+ }
+ if (S_ISDIR(sb.st_mode) == 0) {
+ debug(47, 0) ("%s is not a directory\n", path);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * This function is called by commonUfsDirInit(). If this returns < 0,
+ * then Squid exits, complains about swap directories not
+ * existing, and instructs the admin to run 'squid -z'
+ */
+int
+commonUfsDirVerifyCacheDirs(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ int j;
+ const char *path = sd->path;
+
+ if (commonUfsDirVerifyDirectory(path) < 0)
+ return -1;
+ for (j = 0; j < ioinfo->l1; j++) {
+ path = commonUfsDirSwapSubDir(sd, j);
+ if (commonUfsDirVerifyDirectory(path) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+void
+commonUfsDirCreateSwapSubDirs(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ int i, k;
+ int should_exist;
+ LOCAL_ARRAY(char, name, MAXPATHLEN);
+ for (i = 0; i < ioinfo->l1; i++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
+ if (commonUfsDirCreateDirectory(name, 0))
+ should_exist = 0;
+ else
+ should_exist = 1;
+ debug(47, 1) ("Making directories in %s\n", name);
+ for (k = 0; k < ioinfo->l2; k++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
+ commonUfsDirCreateDirectory(name, should_exist);
+ }
+ }
+}
+
+char *
+commonUfsDirSwapLogFile(SwapDir * sd, const char *ext)
+{
+ LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, digit, 32);
+ char *pathtmp2;
+ if (Config.Log.swap) {
+ xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
+ pathtmp2 = pathtmp;
+ while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
+ *pathtmp2 = '.';
+ while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
+ pathtmp[strlen(pathtmp) - 1] = '\0';
+ for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
+ snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
+ if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
+ strcat(path, ".");
+ snprintf(digit, 32, "%02d", sd->index);
+ strncat(path, digit, 3);
+ }
+ } else {
+ xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
+ strcat(path, "/swap.state");
+ }
+ if (ext)
+ strncat(path, ext, 16);
+ return path;
+}
+
+void
+commonUfsDirOpenSwapLog(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ char *path;
+ int fd;
+ path = commonUfsDirSwapLogFile(sd, NULL);
+ fd = file_open(path, O_WRONLY | O_CREAT | O_BINARY);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", path, xstrerror());
+ fatal("commonUfsDirOpenSwapLog: Failed to open swap log.");
+ }
+ debug(50, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+ ioinfo->swaplog_fd = fd;
+ if (0 == n_dirs)
+ assert(NULL == dir_index);
+ ++n_dirs;
+ assert(n_dirs <= Config.cacheSwap.n_configured);
+}
+
+void
+commonUfsDirCloseSwapLog(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ if (ioinfo->swaplog_fd < 0) /* not open */
+ return;
+ file_close(ioinfo->swaplog_fd);
+ debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
+ sd->index, ioinfo->swaplog_fd);
+ ioinfo->swaplog_fd = -1;
+ n_dirs--;
+ assert(n_dirs >= 0);
+ if (0 == n_dirs)
+ safe_free(dir_index);
+}
+
+void
+commonUfsDirInit(SwapDir * sd)
+{
+ static int started_clean_event = 0;
+ static const char *errmsg =
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
+ commonUfsDirInitBitmap(sd);
+ if (commonUfsDirVerifyCacheDirs(sd) < 0)
+ fatal(errmsg);
+ commonUfsDirOpenSwapLog(sd);
+ commonUfsDirRebuild(sd);
+ if (!started_clean_event) {
+ eventAdd("storeDirClean", commonUfsDirCleanEvent, NULL, 15.0, 1);
+ started_clean_event = 1;
+ }
+ (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize);
+}
+
+void
+commonUfsDirRebuildFromDirectory(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
+ StoreEntry *e = NULL;
+ StoreEntry tmpe;
+ cache_key key[MD5_DIGEST_CHARS];
+ sfileno filn = 0;
+ int count;
+ int size;
+ struct stat sb;
+ int swap_hdr_len;
+ int fd = -1;
+ tlv *tlv_list;
+ tlv *t;
+ assert(rb != NULL);
+ debug(47, 3) ("commonUfsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
+ for (count = 0; count < rb->speed; count++) {
+ assert(fd == -1);
+ fd = commonUfsDirGetNextFile(rb, &filn, &size);
+ if (fd == -2) {
+ debug(47, 1) ("Done scanning %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ store_dirs_rebuilding--;
+ commonUfsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ } else if (fd < 0) {
+ continue;
+ }
+ assert(fd > -1);
+ /* lets get file stats here */
+ if (fstat(fd, &sb) < 0) {
+ debug(47, 1) ("commonUfsDirRebuildFromDirectory: fstat(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(47, 3) (" %s %7d files opened so far.\n",
+ rb->sd->path, rb->counts.scancount);
+ debug(47, 9) ("file_in: fd=%d %08X\n", fd, filn);
+ statCounter.syscalls.disk.reads++;
+ if (FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
+ debug(47, 1) ("commonUfsDirRebuildFromDirectory: read(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ swap_hdr_len = 0;
+#if USE_TRUNCATE
+ if (sb.st_size == 0)
+ continue;
+#endif
+ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
+ if (tlv_list == NULL) {
+ debug(47, 1) ("commonUfsDirRebuildFromDirectory: failed to get meta data\n");
+ /* XXX shouldn't this be a call to commonUfsUnlink ? */
+ commonUfsDirUnlinkFile(SD, filn);
+ continue;
+ }
+ debug(47, 3) ("commonUfsDirRebuildFromDirectory: successful swap meta unpacking\n");
+ memset(key, '\0', MD5_DIGEST_CHARS);
+ memset(&tmpe, '\0', sizeof(StoreEntry));
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_KEY:
+ assert(t->length == MD5_DIGEST_CHARS);
+ xmemcpy(key, t->value, MD5_DIGEST_CHARS);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ default:
+ break;
+ }
+ }
+ storeSwapTLVFree(tlv_list);
+ tlv_list = NULL;
+ if (storeKeyNull(key)) {
+ debug(47, 1) ("commonUfsDirRebuildFromDirectory: NULL key\n");
+ commonUfsDirUnlinkFile(SD, filn);
+ continue;
+ }
+ tmpe.hash.key = key;
+ /* check sizes */
+ if (tmpe.swap_file_sz == 0) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz != sb.st_size) {
+ debug(47, 1) ("commonUfsDirRebuildFromDirectory: SIZE MISMATCH %ld!=%ld\n",
+ (long int) tmpe.swap_file_sz, (long int) sb.st_size);
+ commonUfsDirUnlinkFile(SD, filn);
+ continue;
+ }
+ if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
+ commonUfsDirUnlinkFile(SD, filn);
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(key);
+ if (e && e->lastref >= tmpe.lastref) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (NULL != e) {
+ /* URL already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeRelease(e); /* release old entry */
+ rb->counts.dupcount++;
+ }
+ rb->counts.objcount++;
+ storeEntryDump(&tmpe, 5);
+ e = commonUfsDirAddDiskRestore(SD, key,
+ filn,
+ tmpe.swap_file_sz,
+ tmpe.expires,
+ tmpe.timestamp,
+ tmpe.lastref,
+ tmpe.lastmod,
+ tmpe.refcount, /* refcount */
+ tmpe.flags, /* flags */
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", commonUfsDirRebuildFromDirectory, rb, 0.0, 1);
+}
+
+void
+commonUfsDirRebuildFromSwapLog(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogData s;
+ size_t ss = sizeof(storeSwapLogData);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ debug(47, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ store_dirs_rebuilding--;
+ commonUfsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ }
+ rb->n_read++;
+ if (s.op <= SWAP_LOG_NOP)
+ continue;
+ if (s.op >= SWAP_LOG_MAX)
+ continue;
+ /*
+ * BC: during 2.4 development, we changed the way swap file
+ * numbers are assigned and stored. The high 16 bits used
+ * to encode the SD index number. There used to be a call
+ * to storeDirProperFileno here that re-assigned the index
+ * bits. Now, for backwards compatibility, we just need
+ * to mask it off.
+ */
+ s.swap_filen &= 0x00FFFFFF;
+ debug(47, 3) ("commonUfsDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ if ((e = storeGet(s.key)) != NULL) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ commonUfsDirReplRemove(e);
+ commonUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(47, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFF) == 0) {
+ struct stat sb;
+ if (0 == fstat(fileno(rb->log), &sb))
+ storeRebuildProgress(SD->index,
+ (int) sb.st_size / ss, rb->n_read);
+ }
+ if (!commonUfsDirValidFileno(SD, s.swap_filen, 0)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = commonUfsDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+ commonUfsDirUnrefObj(SD, e);
+ } else {
+ debug_trap("commonUfsDirRebuildFromSwapLog: bad condition");
+ debug(47, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
+ continue;
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ commonUfsDirReplRemove(e);
+ /* Make sure we don't actually unlink the file */
+ commonUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = commonUfsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", commonUfsDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+int
+commonUfsDirGetNextFile(RebuildState * rb, sfileno * filn_p, int *size)
+{
+ SwapDir *SD = rb->sd;
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) SD->fsdata;
+ int fd = -1;
+ int used = 0;
+ int dirs_opened = 0;
+ debug(47, 3) ("commonUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
+ rb->flags.init,
+ rb->sd->index,
+ rb->curlvl1,
+ rb->curlvl2);
+ if (rb->done)
+ return -2;
+ while (fd < 0 && rb->done == 0) {
+ fd = -1;
+ if (0 == rb->flags.init) { /* initialize, open first file */
+ rb->done = 0;
+ rb->curlvl1 = 0;
+ rb->curlvl2 = 0;
+ rb->in_dir = 0;
+ rb->flags.init = 1;
+ assert(Config.cacheSwap.n_configured > 0);
+ }
+ if (0 == rb->in_dir) { /* we need to read in a new directory */
+ snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ rb->sd->path,
+ rb->curlvl1, rb->curlvl2);
+ if (dirs_opened)
+ return -1;
+ rb->td = opendir(rb->fullpath);
+ dirs_opened++;
+ if (rb->td == NULL) {
+ debug(47, 1) ("commonUfsDirGetNextFile: opendir: %s: %s\n",
+ rb->fullpath, xstrerror());
+ } else {
+ rb->entry = readdir(rb->td); /* skip . and .. */
+ rb->entry = readdir(rb->td);
+ if (rb->entry == NULL && errno == ENOENT)
+ debug(47, 1) ("commonUfsDirGetNextFile: directory does not exist!.\n");
+ debug(47, 3) ("commonUfsDirGetNextFile: Directory %s\n", rb->fullpath);
+ }
+ }
+ if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
+ rb->in_dir++;
+ if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
+ debug(47, 3) ("commonUfsDirGetNextFile: invalid %s\n",
+ rb->entry->d_name);
+ continue;
+ }
+ if (!commonUfsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
+ debug(47, 3) ("commonUfsDirGetNextFile: %08X does not belong in %d/%d/%d\n",
+ rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
+ continue;
+ }
+ used = commonUfsDirMapBitTest(SD, rb->fn);
+ if (used) {
+ debug(47, 3) ("commonUfsDirGetNextFile: Locked, continuing with next.\n");
+ continue;
+ }
+ snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
+ rb->fullpath, rb->entry->d_name);
+ debug(47, 3) ("commonUfsDirGetNextFile: Opening %s\n", rb->fullfilename);
+ fd = file_open(rb->fullfilename, O_RDONLY | O_BINARY);
+ if (fd < 0)
+ debug(47, 1) ("commonUfsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
+ else
+ store_open_disk_fd++;
+ continue;
+ }
+ if (rb->td != NULL)
+ closedir(rb->td);
+ rb->td = NULL;
+ rb->in_dir = 0;
+ if (++rb->curlvl2 < ioinfo->l2)
+ continue;
+ rb->curlvl2 = 0;
+ if (++rb->curlvl1 < ioinfo->l1)
+ continue;
+ rb->curlvl1 = 0;
+ rb->done = 1;
+ }
+ *filn_p = rb->fn;
+ return fd;
+}
+
+/* Add a new object to the cache with empty memory copy and pointer to disk
+ * use to rebuild store from disk. */
+StoreEntry *
+commonUfsDirAddDiskRestore(SwapDir * SD, const cache_key * key,
+ sfileno file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_int32_t refcount,
+ u_int16_t flags,
+ int clean)
+{
+ StoreEntry *e = NULL;
+ debug(47, 5) ("commonUfsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
+ /* if you call this you'd better be sure file_number is not
+ * already in use! */
+ e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ e->store_status = STORE_OK;
+ storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->swap_status = SWAPOUT_DONE;
+ e->swap_filen = file_number;
+ e->swap_dirn = SD->index;
+ e->swap_file_sz = swap_file_sz;
+ e->lock_count = 0;
+ e->lastref = lastref;
+ e->timestamp = timestamp;
+ e->expires = expires;
+ e->lastmod = lastmod;
+ e->refcount = refcount;
+ e->flags = flags;
+ EBIT_SET(e->flags, ENTRY_CACHABLE);
+ EBIT_CLR(e->flags, RELEASE_REQUEST);
+ EBIT_CLR(e->flags, KEY_PRIVATE);
+ e->ping_status = PING_NONE;
+ EBIT_CLR(e->flags, ENTRY_VALIDATED);
+ commonUfsDirMapBitSet(SD, e->swap_filen);
+ storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
+ commonUfsDirReplAdd(SD, e);
+ return e;
+}
+
+CBDATA_TYPE(RebuildState);
+
+void
+commonUfsDirRebuild(SwapDir * sd)
+{
+ RebuildState *rb;
+ int clean = 0;
+ int zero = 0;
+ FILE *fp;
+ EVH *func = NULL;
+ CBDATA_INIT_TYPE(RebuildState);
+ rb = cbdataAlloc(RebuildState);
+ rb->sd = sd;
+ rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ /*
+ * If the swap.state file exists in the cache_dir, then
+ * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll
+ * use commonUfsDirRebuildFromDirectory() to open up each file
+ * and suck in the meta data.
+ */
+ fp = commonUfsDirOpenTmpSwapLog(sd, &clean, &zero);
+ if (fp == NULL || zero) {
+ if (fp != NULL)
+ fclose(fp);
+ func = commonUfsDirRebuildFromDirectory;
+ } else {
+ func = commonUfsDirRebuildFromSwapLog;
+ rb->log = fp;
+ rb->flags.clean = (unsigned int) clean;
+ }
+ if (!clean)
+ rb->flags.need_to_validate = 1;
+ debug(47, 1) ("Rebuilding storage in %s (%s)\n",
+ sd->path, clean ? "CLEAN" : "DIRTY");
+ store_dirs_rebuilding++;
+ eventAdd("storeRebuild", func, rb, 0.0, 1);
+}
+
+void
+commonUfsDirCloseTmpSwapLog(SwapDir * sd)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ char *swaplog_path = xstrdup(commonUfsDirSwapLogFile(sd, NULL));
+ char *new_path = xstrdup(commonUfsDirSwapLogFile(sd, ".new"));
+ int fd;
+ file_close(ioinfo->swaplog_fd);
+#if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_) || defined(_SQUID_MSWIN_)
+ if (unlink(swaplog_path) < 0) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("commonUfsDirCloseTmpSwapLog: unlink failed");
+ }
+#endif
+ if (xrename(new_path, swaplog_path) < 0) {
+ fatal("commonUfsDirCloseTmpSwapLog: rename failed");
+ }
+ fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("commonUfsDirCloseTmpSwapLog: Failed to open swap log.");
+ }
+ safe_free(swaplog_path);
+ safe_free(new_path);
+ ioinfo->swaplog_fd = fd;
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+}
+
+FILE *
+commonUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ char *swaplog_path = xstrdup(commonUfsDirSwapLogFile(sd, NULL));
+ char *clean_path = xstrdup(commonUfsDirSwapLogFile(sd, ".last-clean"));
+ char *new_path = xstrdup(commonUfsDirSwapLogFile(sd, ".new"));
+ struct stat log_sb;
+ struct stat clean_sb;
+ FILE *fp;
+ int fd;
+ if (stat(swaplog_path, &log_sb) < 0) {
+ debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return NULL;
+ }
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
+ /* close the existing write-only FD */
+ if (ioinfo->swaplog_fd >= 0)
+ file_close(ioinfo->swaplog_fd);
+ /* open a write-only FD for the new log */
+ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
+ }
+ ioinfo->swaplog_fd = fd;
+ /* open a read-only stream of the old log */
+ fp = fopen(swaplog_path, "rb");
+ if (fp == NULL) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("Failed to open swap log for reading");
+ }
+ memset(&clean_sb, '\0', sizeof(struct stat));
+ if (stat(clean_path, &clean_sb) < 0)
+ *clean_flag = 0;
+ else if (clean_sb.st_mtime < log_sb.st_mtime)
+ *clean_flag = 0;
+ else
+ *clean_flag = 1;
+ safeunlink(clean_path, 1);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return fp;
+}
+
+struct _clean_state {
+ char *cur;
+ char *new;
+ char *cln;
+ char *outbuf;
+ off_t outbuf_offset;
+ int fd;
+ RemovalPolicyWalker *walker;
+};
+
+#define CLEAN_BUF_SZ 16384
+
+/*
+ * Begin the process to write clean cache state. For AUFS this means
+ * opening some log files and allocating write buffers. Return 0 if
+ * we succeed, and assign the 'func' and 'data' return pointers.
+ */
+int
+commonUfsDirWriteCleanStart(SwapDir * sd)
+{
+ struct _clean_state *state = xcalloc(1, sizeof(*state));
+#if HAVE_FCHMOD
+ struct stat sb;
+#endif
+ sd->log.clean.write = NULL;
+ sd->log.clean.state = NULL;
+ state->new = xstrdup(commonUfsDirSwapLogFile(sd, ".clean"));
+ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
+ if (state->fd < 0) {
+ xfree(state->new);
+ xfree(state);
+ return -1;
+ }
+ state->cur = xstrdup(commonUfsDirSwapLogFile(sd, NULL));
+ state->cln = xstrdup(commonUfsDirSwapLogFile(sd, ".last-clean"));
+ state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
+ state->outbuf_offset = 0;
+ state->walker = sd->repl->WalkInit(sd->repl);
+ unlink(state->cln);
+ debug(47, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
+ state->new, state->fd);
+#if HAVE_FCHMOD
+ if (stat(state->cur, &sb) == 0)
+ fchmod(state->fd, sb.st_mode);
+#endif
+ sd->log.clean.write = commonUfsDirWriteCleanEntry;
+ sd->log.clean.state = state;
+ return 0;
+}
+
+/*
+ * Get the next entry that is a candidate for clean log writing
+ */
+const StoreEntry *
+commonUfsDirCleanLogNextEntry(SwapDir * sd)
+{
+ const StoreEntry *entry = NULL;
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->walker)
+ entry = state->walker->Next(state->walker);
+ return entry;
+}
+
+/*
+ * "write" an entry to the clean log file.
+ */
+void
+commonUfsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
+{
+ storeSwapLogData s;
+ static size_t ss = sizeof(storeSwapLogData);
+ struct _clean_state *state = sd->log.clean.state;
+ memset(&s, '\0', ss);
+ s.op = (char) SWAP_LOG_ADD;
+ s.swap_filen = e->swap_filen;
+ s.timestamp = e->timestamp;
+ s.lastref = e->lastref;
+ s.expires = e->expires;
+ s.lastmod = e->lastmod;
+ s.swap_file_sz = e->swap_file_sz;
+ s.refcount = e->refcount;
+ s.flags = e->flags;
+ xmemcpy(&s.key, e->hash.key, MD5_DIGEST_CHARS);
+ xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
+ state->outbuf_offset += ss;
+ /* buffered write */
+ if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
+ if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+ return;
+ }
+ state->outbuf_offset = 0;
+ }
+}
+
+void
+commonUfsDirWriteCleanDone(SwapDir * sd)
+{
+ int fd;
+ struct _clean_state *state = sd->log.clean.state;
+ if (NULL == state)
+ return;
+ if (state->fd < 0)
+ return;
+ state->walker->Done(state->walker);
+ if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(50, 0) ("storeDirWriteCleanLogs: Current swap logfile "
+ "not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ }
+ safe_free(state->outbuf);
+ /*
+ * You can't rename open files on Microsoft "operating systems"
+ * so we have to close before renaming.
+ */
+ commonUfsDirCloseSwapLog(sd);
+ /* save the fd value for a later test */
+ fd = state->fd;
+ /* rename */
+ if (state->fd >= 0) {
+#if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) || defined(_SQUID_MSWIN_)
+ file_close(state->fd);
+ state->fd = -1;
+ if (unlink(state->cur) < 0)
+ debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
+ xstrerror(), state->cur);
+#endif
+ xrename(state->new, state->cur);
+ }
+ /* touch a timestamp file if we're not still validating */
+ if (store_dirs_rebuilding)
+ (void) 0;
+ else if (fd < 0)
+ (void) 0;
+ else
+ file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
+ /* close */
+ safe_free(state->cur);
+ safe_free(state->new);
+ safe_free(state->cln);
+ if (state->fd >= 0)
+ file_close(state->fd);
+ state->fd = -1;
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+}
+
+void
+storeSwapLogDataFree(void *s)
+{
+ memFree(s, MEM_SWAP_LOG_DATA);
+}
+
+void
+commonUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) sd->fsdata;
+ storeSwapLogData *s = memAllocate(MEM_SWAP_LOG_DATA);
+ s->op = (char) op;
+ s->swap_filen = e->swap_filen;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->swap_file_sz = e->swap_file_sz;
+ s->refcount = e->refcount;
+ s->flags = e->flags;
+ xmemcpy(s->key, e->hash.key, MD5_DIGEST_CHARS);
+ file_write(ioinfo->swaplog_fd,
+ -1,
+ s,
+ sizeof(storeSwapLogData),
+ NULL,
+ NULL,
+ (FREE *) storeSwapLogDataFree);
+}
+
+void
+commonUfsDirNewfs(SwapDir * sd)
+{
+ debug(47, 3) ("Creating swap space in %s\n", sd->path);
+ commonUfsDirCreateDirectory(sd->path, 0);
+ commonUfsDirCreateSwapSubDirs(sd);
+}
+
+static int
+rev_int_sort(const void *A, const void *B)
+{
+ const int *i1 = A;
+ const int *i2 = B;
+ return *i2 - *i1;
+}
+
+int
+commonUfsDirClean(int swap_index)
+{
+ DIR *dp = NULL;
+ struct dirent *de = NULL;
+ LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
+ LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
+#if USE_TRUNCATE
+ struct stat sb;
+#endif
+ int files[20];
+ int swapfileno;
+ int fn; /* same as swapfileno, but with dirn bits set */
+ int n = 0;
+ int k = 0;
+ int N0, N1, N2;
+ int D0, D1, D2;
+ SwapDir *SD;
+ squidufsinfo_t *ioinfo;
+ N0 = n_dirs;
+ D0 = dir_index[swap_index % N0];
+ SD = &Config.cacheSwap.swapDirs[D0];
+ ioinfo = (squidufsinfo_t *) SD->fsdata;
+ N1 = ioinfo->l1;
+ D1 = (swap_index / N0) % N1;
+ N2 = ioinfo->l2;
+ D2 = ((swap_index / N0) / N1) % N2;
+ snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ Config.cacheSwap.swapDirs[D0].path, D1, D2);
+ debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
+ dp = opendir(p1);
+ if (dp == NULL) {
+ if (errno == ENOENT) {
+ debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
+ if (mkdir(p1, 0777) == 0)
+ return 0;
+ }
+ debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
+ safeunlink(p1, 1);
+ return 0;
+ }
+ while ((de = readdir(dp)) != NULL && k < 20) {
+ if (sscanf(de->d_name, "%X", &swapfileno) != 1)
+ continue;
+ fn = swapfileno; /* XXX should remove this cruft ! */
+ if (commonUfsDirValidFileno(SD, fn, 1))
+ if (commonUfsDirMapBitTest(SD, fn))
+ if (commonUfsFilenoBelongsHere(fn, D0, D1, D2))
+ continue;
+#if USE_TRUNCATE
+ if (!stat(de->d_name, &sb))
+ if (sb.st_size == 0)
+ continue;
+#endif
+ files[k++] = swapfileno;
+ }
+ closedir(dp);
+ if (k == 0)
+ return 0;
+ qsort(files, k, sizeof(int), rev_int_sort);
+ if (k > 10)
+ k = 10;
+ for (n = 0; n < k; n++) {
+ debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
+ snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
+#if USE_TRUNCATE
+ truncate(p2, 0);
+#else
+ safeunlink(p2, 0);
+#endif
+ statCounter.swap.files_cleaned++;
+ }
+ debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
+ return k;
+}
+
+void
+commonUfsDirCleanEvent(void *unused)
+{
+ static int swap_index = 0;
+ int i;
+ int j = 0;
+ int n = 0;
+ /*
+ * Assert that there are AUFS cache_dirs configured, otherwise
+ * we should never be called.
+ */
+ assert(n_dirs);
+ if (NULL == dir_index) {
+ SwapDir *sd;
+ squidufsinfo_t *ioinfo;
+ /*
+ * Initialize the little array that translates AUFS cache_dir
+ * number into the Config.cacheSwap.swapDirs array index.
+ */
+ dir_index = xcalloc(n_dirs, sizeof(*dir_index));
+ for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
+ sd = &Config.cacheSwap.swapDirs[i];
+ if (!commonUfsDirIs(sd))
+ continue;
+ dir_index[n++] = i;
+ ioinfo = (squidufsinfo_t *) sd->fsdata;
+ j += (ioinfo->l1 * ioinfo->l2);
+ }
+ assert(n == n_dirs);
+ /*
+ * Start the commonUfsDirClean() swap_index with a random
+ * value. j equals the total number of AUFS level 2
+ * swap directories
+ */
+ swap_index = (int) (squid_random() % j);
+ }
+ if (0 == store_dirs_rebuilding) {
+ n = commonUfsDirClean(swap_index);
+ swap_index++;
+ }
+ eventAdd("storeDirClean", commonUfsDirCleanEvent, NULL,
+ 15.0 * exp(-0.25 * n), 1);
+}
+
+int
+commonUfsDirIs(SwapDir * sd)
+{
+ if (strncmp(sd->type, "aufs", 4) == 0)
+ return 1;
+ if (strncmp(sd->type, "diskd", 5) == 0)
+ return 1;
+ if (strncmp(sd->type, "ufs", 3) == 0)
+ return 1;
+ return 0;
+}
+
+/*
+ * Does swapfile number 'fn' belong in cachedir #F0,
+ * level1 dir #F1, level2 dir #F2?
+ */
+int
+commonUfsFilenoBelongsHere(int fn, int F0, int F1, int F2)
+{
+ int D1, D2;
+ int L1, L2;
+ int filn = fn;
+ squidufsinfo_t *ioinfo;
+ assert(F0 < Config.cacheSwap.n_configured);
+ ioinfo = (squidufsinfo_t *) Config.cacheSwap.swapDirs[F0].fsdata;
+ L1 = ioinfo->l1;
+ L2 = ioinfo->l2;
+ D1 = ((filn / L2) / L2) % L1;
+ if (F1 != D1)
+ return 0;
+ D2 = (filn / L2) % L2;
+ if (F2 != D2)
+ return 0;
+ return 1;
+}
+
+int
+commonUfsDirValidFileno(SwapDir * SD, sfileno filn, int flag)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) SD->fsdata;
+ if (filn < 0)
+ return 0;
+ /*
+ * If flag is set it means out-of-range file number should
+ * be considered invalid.
+ */
+ if (flag)
+ if (filn > ioinfo->map->max_n_files)
+ return 0;
+ return 1;
+}
+
+void
+commonUfsDirMaintain(SwapDir * SD)
+{
+ StoreEntry *e = NULL;
+ int removed = 0;
+ int max_scan;
+ int max_remove;
+ double f;
+ RemovalPurgeWalker *walker;
+ /* We can't delete objects while rebuilding swap */
+ if (store_dirs_rebuilding) {
+ return;
+ } else {
+ f = (double) (SD->cur_size - SD->low_size) / (SD->max_size - SD->low_size);
+ f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
+ max_scan = (int) (f * 400.0 + 100.0);
+ max_remove = (int) (f * 70.0 + 10.0);
+ /*
+ * This is kinda cheap, but so we need this priority hack?
+ */
+ }
+ debug(47, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n",
+ f, max_scan, max_remove);
+ walker = SD->repl->PurgeInit(SD->repl, max_scan);
+ while (1) {
+ if (SD->cur_size < SD->low_size)
+ break;
+ if (removed >= max_remove)
+ break;
+ e = walker->Next(walker);
+ if (!e)
+ break; /* no more objects */
+ removed++;
+ storeRelease(e);
+ }
+ walker->Done(walker);
+ debug(47, (removed ? 2 : 3)) ("commonUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
+ SD->path, removed, max_remove, f, max_scan);
+}
+
+#if 0
+/*
+ * commonUfsDirCheckObj
+ *
+ * This routine is called by storeDirSelectSwapDir to see if the given
+ * object is able to be stored on this filesystem. AUFS filesystems will
+ * happily store anything as long as the LRU time isn't too small.
+ */
+int
+commonUfsDirCheckObj(SwapDir * SD, const StoreEntry * e)
+{
+ int loadav;
+ int ql;
+
+#if OLD_UNUSED_CODE
+ if (commonUfsDirExpiredReferenceAge(SD) < 300) {
+ debug(47, 3) ("commonUfsDirCheckObj: NO: LRU Age = %d\n",
+ commonUfsDirExpiredReferenceAge(SD));
+ /* store_check_cachable_hist.no.lru_age_too_low++; */
+ return -1;
+ }
+#endif
+ ql = aioQueueSize();
+ if (ql == 0)
+ loadav = 0;
+ loadav = ql * 1000 / MAGIC1;
+ debug(47, 9) ("commonUfsDirCheckObj: load=%d\n", loadav);
+ return loadav;
+}
+#endif
+/*
+ * commonUfsDirRefObj
+ *
+ * This routine is called whenever an object is referenced, so we can
+ * maintain replacement information within the storage fs.
+ */
+void
+commonUfsDirRefObj(SwapDir * SD, StoreEntry * e)
+{
+ debug(47, 3) ("commonUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
+ e->swap_filen);
+ if (SD->repl->Referenced)
+ SD->repl->Referenced(SD->repl, e, &e->repl);
+}
+
+/*
+ * commonUfsDirUnrefObj
+ * This routine is called whenever the last reference to an object is
+ * removed, to maintain replacement information within the storage fs.
+ */
+void
+commonUfsDirUnrefObj(SwapDir * SD, StoreEntry * e)
+{
+ debug(47, 3) ("commonUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
+ e->swap_filen);
+ if (SD->repl->Dereferenced)
+ SD->repl->Dereferenced(SD->repl, e, &e->repl);
+}
+
+/*
+ * commonUfsDirUnlinkFile
+ *
+ * This routine unlinks a file and pulls it out of the bitmap.
+ * It used to be in commonUfsUnlink(), however an interface change
+ * forced this bit of code here. Eeek.
+ */
+void
+commonUfsDirUnlinkFile(SwapDir * SD, sfileno f)
+{
+ squidufsinfo_t *ioinfo = SD->fsdata;
+ debug(79, 3) ("commonUfsDirUnlinkFile: unlinking fileno %08X\n", f);
+ /* commonUfsDirMapBitReset(SD, f); */
+ assert(ioinfo->io.storeDirUnlinkFile);
+ ioinfo->io.storeDirUnlinkFile(commonUfsDirFullPath(SD, f, NULL));
+}
+
+
+/*
+ * Add and remove the given StoreEntry from the replacement policy in
+ * use.
+ */
+
+void
+commonUfsDirReplAdd(SwapDir * SD, StoreEntry * e)
+{
+ debug(47, 4) ("commonUfsDirReplAdd: added node %p to dir %d\n", e,
+ SD->index);
+ SD->repl->Add(SD->repl, e, &e->repl);
+}
+
+
+void
+commonUfsDirReplRemove(StoreEntry * e)
+{
+ SwapDir *SD;
+ if (e->swap_dirn < 0)
+ return;
+ SD = INDEXSD(e->swap_dirn);
+ debug(47, 4) ("commonUfsDirReplRemove: remove node %p from dir %d\n", e,
+ SD->index);
+ SD->repl->Remove(SD->repl, e, &e->repl);
+}
+
+
+
+/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
+
+void
+commonUfsDirStats(SwapDir * SD, StoreEntry * sentry)
+{
+ squidufsinfo_t *ioinfo = SD->fsdata;
+ int totl_kb = 0;
+ int free_kb = 0;
+ int totl_in = 0;
+ int free_in = 0;
+ int x;
+ storeAppendPrintf(sentry, "First level subdirectories: %d\n", ioinfo->l1);
+ storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ioinfo->l2);
+ storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
+ storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
+ storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
+ 100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
+ ioinfo->map->n_files_in_map, ioinfo->map->max_n_files,
+ percent(ioinfo->map->n_files_in_map, ioinfo->map->max_n_files));
+ x = storeDirGetUFSStats(SD->path, &totl_kb, &free_kb, &totl_in, &free_in);
+ if (0 == x) {
+ storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
+ totl_kb - free_kb,
+ totl_kb,
+ percent(totl_kb - free_kb, totl_kb));
+ storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
+ totl_in - free_in,
+ totl_in,
+ percent(totl_in - free_in, totl_in));
+ }
+ storeAppendPrintf(sentry, "Flags:");
+ if (SD->flags.selected)
+ storeAppendPrintf(sentry, " SELECTED");
+ if (SD->flags.read_only)
+ storeAppendPrintf(sentry, " READ-ONLY");
+ storeAppendPrintf(sentry, "\n");
+}
+
+#if 0
+static struct cache_dir_option options[] =
+{
+#if NOT_YET_DONE
+ {"L1", commonUfsDirParseL1, commonUfsDirDumpL1},
+ {"L2", commonUfsDirParseL2, commonUfsDirDumpL2},
+#endif
+ {NULL, NULL}
+};
+
+/*
+ * commonUfsDirReconfigure
+ *
+ * This routine is called when the given swapdir needs reconfiguring
+ */
+static void
+commonUfsDirReconfigure(SwapDir * sd, int index, char *path)
+{
+ int i;
+ int size;
+ int l1;
+ int l2;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("commonUfsDirReconfigure: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("commonUfsDirReconfigure: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("commonUfsDirReconfigure: invalid level 2 directories value");
+
+ /* just reconfigure it */
+ if (size == sd->max_size)
+ debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n",
+ path, size);
+ else
+ debug(3, 1) ("Cache dir '%s' size changed to %d KB\n",
+ path, size);
+ sd->max_size = size;
+
+ parse_cachedir_options(sd, options, 0);
+
+ return;
+}
+
+#endif
+
+void
+commonUfsDirDump(StoreEntry * entry, SwapDir * s)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) s->fsdata;
+ storeAppendPrintf(entry, " %d %d %d",
+ s->max_size >> 10,
+ ioinfo->l1,
+ ioinfo->l2);
+}
+
+/*
+ * Only "free" the filesystem specific stuff here
+ */
+void
+commonUfsDirFree(SwapDir * s)
+{
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) s->fsdata;
+ if (ioinfo->swaplog_fd > -1) {
+ file_close(ioinfo->swaplog_fd);
+ ioinfo->swaplog_fd = -1;
+ }
+ filemapFreeMemory(ioinfo->map);
+ xfree(ioinfo);
+ s->fsdata = NULL; /* Will aid debugging... */
+}
+
+
+char *
+commonUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath)
+{
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ squidufsinfo_t *ioinfo = (squidufsinfo_t *) SD->fsdata;
+ int L1 = ioinfo->l1;
+ int L2 = ioinfo->l2;
+ if (!fullpath)
+ fullpath = fullfilename;
+ fullpath[0] = '\0';
+ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
+ SD->path,
+ ((filn / L2) / L2) % L1,
+ (filn / L2) % L2,
+ filn);
+ return fullpath;
+}
+
+/*
+ * commonUfsCleanupDoubleCheck
+ *
+ * This is called by storeCleanup() if -S was given on the command line.
+ */
+int
+commonUfsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
+{
+ struct stat sb;
+ if (stat(commonUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: MISSING SWAP FILE\n");
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: PATH %s\n",
+ commonUfsDirFullPath(sd, e->swap_filen, NULL));
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ if (e->swap_file_sz != sb.st_size) {
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: SIZE MISMATCH\n");
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: PATH %s\n",
+ commonUfsDirFullPath(sd, e->swap_filen, NULL));
+ debug(47, 0) ("commonUfsCleanupDoubleCheck: ENTRY SIZE: %ld, FILE SIZE: %ld\n",
+ (long int) e->swap_file_sz, (long int) sb.st_size);
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ return 0;
+}
+
+#if 0
+/*
+ * commonUfsDirParse *
+ * Called when a *new* fs is being setup.
+ */
+static void
+commonUfsDirParse(SwapDir * sd, int index, char *path)
+{
+ int i;
+ int size;
+ int l1;
+ int l2;
+ squidufsinfo_t *ioinfo;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("commonUfsDirParse: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("commonUfsDirParse: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("commonUfsDirParse: invalid level 2 directories value");
+
+ ioinfo = xmalloc(sizeof(squidufsinfo_t));
+ if (ioinfo == NULL)
+ fatal("commonUfsDirParse: couldn't xmalloc() squidufsinfo_t!\n");
+
+ sd->index = index;
+ sd->path = xstrdup(path);
+ sd->max_size = size;
+ sd->fsdata = ioinfo;
+ ioinfo->l1 = l1;
+ ioinfo->l2 = l2;
+ ioinfo->swaplog_fd = -1;
+ ioinfo->map = NULL; /* Debugging purposes */
+ ioinfo->suggest = 0;
+ sd->init = commonUfsDirInit;
+ sd->newfs = commonUfsDirNewfs;
+ sd->dump = commonUfsDirDump;
+ sd->freefs = commonUfsDirFree;
+ sd->dblcheck = commonUfsCleanupDoubleCheck;
+ sd->statfs = commonUfsDirStats;
+ sd->maintainfs = commonUfsDirMaintain;
+ sd->checkobj = commonUfsDirCheckObj;
+ sd->refobj = commonUfsDirRefObj;
+ sd->unrefobj = commonUfsDirUnrefObj;
+ sd->callback = aioCheckCallbacks;
+ sd->sync = aioSync;
+ sd->obj.create = commonUfsCreate;
+ sd->obj.open = commonUfsOpen;
+ sd->obj.close = commonUfsClose;
+ sd->obj.read = commonUfsRead;
+ sd->obj.write = commonUfsWrite;
+ sd->obj.unlink = commonUfsUnlink;
+ sd->log.open = commonUfsDirOpenSwapLog;
+ sd->log.close = commonUfsDirCloseSwapLog;
+ sd->log.write = commonUfsDirSwapLog;
+ sd->log.clean.start = commonUfsDirWriteCleanStart;
+ sd->log.clean.nextentry = commonUfsDirCleanLogNextEntry;
+ sd->log.clean.done = commonUfsDirWriteCleanDone;
+
+ parse_cachedir_options(sd, options, 0);
+
+ /* Initialise replacement policy stuff */
+ sd->repl = createRemovalPolicy(Config.replPolicy);
+}
+
+/*
+ * Initial setup / end destruction
+ */
+static void
+commonUfsDirDone(void)
+{
+ aioDone();
+ memPoolDestroy(&squidaio_state_pool);
+ memPoolDestroy(&aufs_qread_pool);
+ memPoolDestroy(&aufs_qwrite_pool);
+ asyncufs_initialised = 0;
+}
+
+void
+storeFsSetup_aufs(storefs_entry_t * storefs)
+{
+ assert(!asyncufs_initialised);
+ storefs->parsefunc = commonUfsDirParse;
+ storefs->reconfigurefunc = commonUfsDirReconfigure;
+ storefs->donefunc = commonUfsDirDone;
+ squidaio_state_pool = memPoolCreate("AUFS IO State data", sizeof(squidaiostate_t));
+ aufs_qread_pool = memPoolCreate("AUFS Queued read data",
+ sizeof(queued_read));
+ aufs_qwrite_pool = memPoolCreate("AUFS Queued write data",
+ sizeof(queued_write));
+
+ asyncufs_initialised = 1;
+ aioInit();
+}
+#endif
--- /dev/null
+
+/*
+ * $Id: ufscommon.h,v 1.1 2002/10/12 09:45:56 robertc Exp $
+ *
+ * SQUID Web Proxy Cache http://www.squid-cache.org/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from
+ * the Internet community; see the CONTRIBUTORS file for full
+ * details. Many organizations have provided support for Squid's
+ * development; see the SPONSORS file for full details. Squid is
+ * Copyrighted (C) 2001 by the Regents of the University of
+ * California; see the COPYRIGHT file for full details. Squid
+ * incorporates software developed and/or copyrighted by other
+ * sources; see the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#ifndef SQUID_UFSCOMMON_H
+#define SQUID_UFSCOMMON_H
+
+#include "squid.h"
+
+#define DefaultLevelOneDirs 16
+#define DefaultLevelTwoDirs 256
+#define STORE_META_BUFSZ 4096
+
+typedef struct _iospecific_t iospecific_t;
+struct _iospecific_t {
+ void (*storeDirUnlinkFile) (char *);
+};
+
+typedef struct _squidufsinfo_t squidufsinfo_t;
+struct _squidufsinfo_t {
+ int swaplog_fd;
+ int l1;
+ int l2;
+ fileMap *map;
+ int suggest;
+ iospecific_t io;
+};
+
+/* Common UFS routines */
+void commonUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op);
+FREE storeSwapLogDataFree;
+void commonUfsDirWriteCleanDone(SwapDir * sd);
+const StoreEntry *commonUfsDirCleanLogNextEntry(SwapDir * sd);
+void commonUfsDirCloseSwapLog(SwapDir * sd);
+int commonUfsDirWriteCleanStart(SwapDir * sd);
+void commonUfsDirInit(SwapDir * sd);
+void commonUfsDirUnlinkFile(SwapDir * SD, sfileno f);
+void commonUfsDirOpenSwapLog(SwapDir * sd);
+void commonUfsDirNewfs(SwapDir * sd);
+void commonUfsDirMaintain(SwapDir * SD);
+void commonUfsDirRefObj(SwapDir * SD, StoreEntry * e);
+void commonUfsDirUnrefObj(SwapDir * SD, StoreEntry * e);
+void commonUfsDirReplAdd(SwapDir * SD, StoreEntry * e);
+void commonUfsDirReplRemove(StoreEntry * e);
+void commonUfsDirStats(SwapDir * SD, StoreEntry * sentry);
+void commonUfsDirDump(StoreEntry * entry, SwapDir * s);
+void commonUfsDirFree(SwapDir * s);
+char *commonUfsDirFullPath(SwapDir * SD, sfileno filn, char *fullpath);
+int commonUfsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e);
+int commonUfsDirMapBitAllocate(SwapDir * SD);
+void commonUfsDirMapBitReset(SwapDir * SD, sfileno filn);
+
+#endif /* SQUID_UFSCOMMON_H */
projects / thirdparty / squid.git / commitdiff
