This also includes Adrians improved store I/O callback handler model.
dnl
dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9)
dnl
-dnl $Id: configure.in,v 1.192 2000/05/31 08:57:08 hno Exp $
+dnl $Id: configure.in,v 1.193 2000/06/08 18:05:30 hno Exp $
dnl
dnl
dnl
AC_INIT(src/main.c)
AC_CONFIG_HEADER(include/autoconf.h)
-AC_REVISION($Revision: 1.192 $)dnl
+AC_REVISION($Revision: 1.193 $)dnl
AC_PREFIX_DEFAULT(/usr/local/squid)
AC_CONFIG_AUX_DIR(cfgaux)
STORE_OBJS="fs/`echo $STORE_MODULES|sed -e's% %.a fs/%g'`.a"
AC_SUBST(STORE_OBJS)
+dnl --enable-heap-replacement compability option
+AC_ARG_ENABLE(heap-replacement,
+[ --enable-heap-replacement
+ Backwards compability option. Please use the
+ new --enable-removal-policies directive instead.],
+[ if test "$enableval" = "yes" ; then
+ echo "--enable-heap-replacement is obsolete. please use the new"
+ echo "--enable-removal-policies directive instead"
+ sleep 5
+ REPL_POLICIES="heap"
+ fi
+])
+
+AC_ARG_ENABLE(removal_policies,
+[ --enable-removal-policies=\"list of policies\"
+ Build support for the list of removal policies.
+ The default is only to build the "lru" module.
+ See src/repl for a list of available modules, or
+ Programmers Guide section 9.9 for details on how
+ to build your custom policy],
+[ case $enableval in
+ yes)
+ for module in $srcdir/src/repl/*; do
+ if test -f $module/Makefile.in; then
+ REPL_POLICIES="$REPL_POLICIES `basename $module`"
+ fi
+ done
+ ;;
+ no)
+ ;;
+ *) REPL_POLICIES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`"
+ ;;
+ esac
+],
+[ if test -z "$REPL_POLICIES"; then
+ REPL_POLICIES="lru"
+ fi
+])
+echo "Store modules built: $REPL_POLICIES"
+AC_SUBST(REPL_POLICIES)
+REPL_OBJS="repl/`echo $REPL_POLICIES|sed -e's% %.a repl/%g'`.a"
+AC_SUBST(REPL_OBJS)
+
OPT_PINGER_EXE=''
AC_ARG_ENABLE(icmp,
[ --enable-icmp Enable ICMP pinging],
fi
])
-dnl Enable HEAP_REPLACEMENT
-AC_ARG_ENABLE(heap-replacement,
-[ --enable-heap-replacement
- This option allows you to use various cache
- replacement algorithms, instead of the standard
- LRU algorithm.],
-[ if test "$enableval" = "yes" ; then
- echo "Enabling HEAP_REPLACEMENT"
- AC_DEFINE(HEAP_REPLACEMENT, 1)
- fi
-])
-
dnl Select auth modules to build
AUTH_MODULES=
AC_ARG_ENABLE(auth-modules,
./src/fs/ufs/Makefile \
./src/fs/aufs/Makefile \
./src/fs/coss/Makefile \
+ ./src/repl/Makefile \
+ ./src/repl/lru/Makefile \
+ ./src/repl/heap/Makefile \
./src/fs/diskd/Makefile \
./contrib/Makefile \
$SNMP_MAKEFILE \
<article>
<title>Squid Programmers Guide</title>
<author>Duane Wessels, Squid Developers
-<date>$Id: prog-guide.sgml,v 1.27 2000/05/03 17:15:40 adrian Exp $</date>
+<date>$Id: prog-guide.sgml,v 1.28 2000/06/08 18:05:33 hno Exp $</date>
<abstract>
Squid is a WWW Cache application developed by the National Laboratory
This feature may not be required by some storage systems
and can be implemented as a null-function (no-op).
-<sect2><tt/log.clean.open()/
+<sect2><tt/log.clean.start()/
<P>
<verb>
int
- STLOGCLEANOPEN(SwapDir *);
+ STLOGCLEANSTART(SwapDir *);
</verb>
<P>
- The <tt/log.clean.open/ function, of type <em/STLOGCLEANOPEN/,
+ The <tt/log.clean.start/ function, of type <em/STLOGCLEANSTART/,
is used for the process of writing "clean" state-holding
log files. The clean-writing procedure is initiated by
the <em/squid -k rotate/ command. This is a special case
keep state information for the clean-writing process, but
should not be accessed by upper layers.
+<sect2><tt/log.clean.nextentry()/
+
+ <P>
+<verb>
+ StoreEntry *
+ STLOGCLEANNEXTENTRY(SwapDir *);
+</verb>
+
+ <P>
+ Gets the next entry that is a candidate for the clean log.
+
+ <P>
+ Returns NULL when there is no more objects to log
+
<sect2><tt/log.clean.write()/
<P>
<verb>
void
- STLOGCLEANWRITE(const StoreEntry *, SwapDir *);
+ STLOGCLEANWRITE(SwapDir *, const StoreEntry *);
</verb>
<P>
The <tt/log.clean.write/ function, of type <em/STLOGCLEANWRITE/,
writes an entry to the clean log file (if any).
+<sect2><tt/log.clean.done()/
+
<P>
- A NULL <em/StoreEntry/ argument indicates the end of
- the clean-writing process and signals the storage
- system to close the clean log, and rename or move them
- to become the official state-holding log.
+<verb>
+ void
+ STLOGCLEANDONE(SwapDir *);
+</verb>
+ <P>
+ Indicates the end of the clean-writing process and signals
+ the storage system to close the clean log, and rename or
+ move them to become the official state-holding log ready
+ to be opened.
<sect1>Replacement policy implementation
coupling between the storage layer and the replacement policy.
-<sect1>Future removal policy
-
- <P>
- (replaces the above Replace policy)
+<sect1>Removal policy API
<P>
The removal policy is responsible for determining in which order
</verb>
<P>
- Tells the policy that a StoreEntry has been referenced.
+ Tells the policy that a StoreEntry is going to be referenced. Called
+ whenever a entry gets locked.
<P>
- datap is a pointer to where policy specific data is stored
+ node is a pointer to where policy specific data is stored
+ for the store entry, currently the size of one (void *) pointer.
+
+<sect3>policy.Dereferenced()
+
+<P>
+<verb>
+ policy->Dereferenced(RemovalPolicy *policy, const StoreEntry *, RemovalPolicyNode *node)
+</verb>
+
+ <P>
+ Tells the policy that a StoreEntry has been referenced. Called when
+ an access to the entry has finished.
+
+ <P>
+ node is a pointer to where policy specific data is stored
for the store entry, currently the size of one (void *) pointer.
<sect3>policy.WalkInit()
Prior to returning the created instance must be registered as
callback-data by calling cbdataAdd().
+<sect2>Design notes/bugs
+
+<P>
+ The RemovalPolicyNode design is incomplete/insufficient. The intention
+ was to abstract the location of the index pointers from the policy
+ implementation to allow the policy to work on both on-disk and memory
+ caches, but unfortunately the purge method for HEAP based policies
+ needs to update this, and it is also preferable if the purge method
+ in general knows how to clear the information. I think the agreement
+ was that the current design of thightly coupling the two together
+ on one StoreEntry is not the best design possible.
+
+<P>
+ It is debated if the design in having the policy index control the
+ clean index writes is the correct approach. Perhaps not. Perhaps a
+ more appropriate design is probably to do the store indexing
+ completely outside the policy implementation (i.e. using the hash
+ index), and only ask the policy to dump it's state somehow.
+
+<P>
+ The Referenced/Dereferenced() calls is today mapped to lock/unlock
+ which is an approximation of when they are intended to be called.
+ However, the real intention is to have Referenced() called whenever
+ an object is referenced, and Dereferenced() only called when the
+ object has actually been used for anything good.
+
<!-- %%%% Chapter : FORWARDING SELECTION %%%% -->
<sect>Forwarding Selection
section 78 HTTP Connection Header
section 79 HTTP Meter Header
section 80 WCCP
+section 81 Store Removal/Replacement policy
#
# Makefile for the Squid Object Cache server
#
-# $Id: Makefile.in,v 1.188 2000/05/31 08:57:08 hno Exp $
+# $Id: Makefile.in,v 1.189 2000/06/08 18:05:34 hno Exp $
#
# Uncomment and customize the following to suit your needs:
#
srcdir = @srcdir@
VPATH = @srcdir@
-SUBDIRS = fs
+SUBDIRS = fs repl
# Gotta love the DOS legacy
#
XTRA_OBJS = @XTRA_OBJS@
STORE_OBJS = @STORE_OBJS@
STORE_MODULES = @STORE_MODULES@
+REPL_OBJS = @REPL_OBJS@
+REPL_POLICIES = @REPL_POLICIES@
MV = @MV@
RM = @RM@
SHELL = /bin/sh
pump.o \
redirect.o \
refresh.o \
+ repl_modules.o \
send-announce.o \
@SNMP_OBJS@ \
ssl.o \
store_swapin.o \
store_swapmeta.o \
store_swapout.o \
- store_heap_replacement.o \
string_arrays.o \
tools.o \
@UNLINKD_OBJS@ \
DEFAULTS = \
-DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\"
-all: squid.conf store_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS)
+all: squid.conf store_modules repl_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS)
$(OBJS): $(top_srcdir)/include/version.h ../include/autoconf.h
$(SNMP_OBJS): ../snmplib/libsnmp.a $(top_srcdir)/include/cache_snmp.h
-$(SQUID_EXE): $(OBJS) $(STORE_OBJS)
- $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(SQUID_LIBS)
+$(SQUID_EXE): $(OBJS) $(STORE_OBJS) $(REPL_OBJS)
+ $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(REPL_OBJS) $(SQUID_LIBS)
globals.o: globals.c Makefile
$(CC) -c globals.c $(CFLAGS) -I$(srcdir) $(DEFAULTS)
< $(srcdir)/cf.data.pre >$@
store_modules.c: store_modules.sh Makefile
- @sh $(srcdir)/store_modules.sh $(STORE_MODULES) >store_modules.c
+ sh $(srcdir)/store_modules.sh $(STORE_MODULES) >store_modules.c
store_modules.o: store_modules.c
$(CC) -c store_modules.c $(CFLAGS) -I$(srcdir)
store_modules fs/stamp:
@sh -c "cd fs && $(MAKE) all"
+repl_modules.c: repl_modules.sh Makefile
+ sh $(srcdir)/repl_modules.sh $(REPL_POLICIES) >repl_modules.c
+
+repl_modules.o: repl_modules.c
+ $(CC) -c repl_modules.c $(CFLAGS) -I$(srcdir)
+
+$(REPL_OBJS): repl/stamp
+
+repl_modules repl/stamp:
+ @sh -c "cd repl && $(MAKE) all"
+
install-mkdirs:
-@if test ! -d $(prefix); then \
echo "mkdir $(prefix)"; \
/*
- * $Id: cache_cf.cc,v 1.349 2000/05/31 07:01:41 hno Exp $
+ * $Id: cache_cf.cc,v 1.350 2000/06/08 18:05:34 hno Exp $
*
* DEBUG: section 3 Configuration File Parsing
* AUTHOR: Harvest Derived
debug(3, 0) ("WARNING: resetting 'maximum_single_addr_tries to 1\n");
Config.retry.maxtries = 1;
}
-#if HEAP_REPLACEMENT
- /* The non-LRU policies do not use referenceAge */
-#else
- if (Config.referenceAge < 300) {
- debug(3, 0) ("WARNING: resetting 'reference_age' to 1 week\n");
- Config.referenceAge = 86400 * 7;
- }
-#endif
requirePathnameExists("MIME Config Table", Config.mimeTablePathname);
#if USE_DNSSERVERS
requirePathnameExists("cache_dns_program", Config.Program.dnsserver);
storeAppendPrintf(entry, "%s %s\n", name, s);
}
+static void
+free_removalpolicy(RemovalPolicySettings **settings)
+{
+ if (!*settings)
+ return;
+ free_string(&(*settings)->type);
+ free_wordlist(&(*settings)->args);
+ xfree(*settings);
+ *settings = NULL;
+}
+
+static void
+parse_removalpolicy(RemovalPolicySettings **settings)
+{
+ if (*settings)
+ free_removalpolicy(settings);
+ *settings = xcalloc(1, sizeof(**settings));
+ parse_string(&(*settings)->type);
+ parse_wordlist(&(*settings)->args);
+}
+
+static void
+dump_removalpolicy(StoreEntry * entry, const char *name, RemovalPolicySettings *settings)
+{
+ wordlist *args;
+ storeAppendPrintf(entry, "%s %s", name, settings->type);
+ args = settings->args;
+ while (args) {
+ storeAppendPrintf(entry, " %s", args->key);
+ args = args->next;
+ }
+}
+
+
#include "cf_parser.c"
peer_t
if (stat(path, &sb) < 0)
fatalf("%s: %s", path, xstrerror());
}
+
+
#
-# $Id: cf.data.pre,v 1.187 2000/05/31 07:01:42 hno Exp $
+# $Id: cf.data.pre,v 1.188 2000/06/08 18:05:35 hno Exp $
#
#
# SQUID Internet Object Cache http://squid.nlanr.net/Squid/
Maximum number of FQDN cache entries.
DOC_END
+NAME: cache_replacement_policy
+TYPE: removalpolicy
+LOC: Config.replPolicy
+DEFAULT: lru
+DOC_START
+ The cache replacement policy parameter determines which
+ objects are evicted (replaced) when disk space is needed.
+
+ lru : Squid's original list based LRU policy
+ heap GDSF : Greedy-Dual Size Frequency
+ heap LFUDA: Least Frequently Used with Dynamic Aging
+ heap LRU : LRU policy implemented using a heap
+
+ Applies to any cache_dir lines listed below this.
+
+ The LRU policies keeps recently referenced objects.
+
+ The heap GDSF policy optimizes object hit rate by keeping smaller
+ popular objects in cache so it has a better chance of getting a
+ hit. It achieves a lower byte hit rate than LFUDA though since
+ it evicts larger (possibly popular) objects.
+
+ The heap LFUDA policy keeps popular objects in cache regardless of
+ their size and thus optimizes byte hit rate at the expense of
+ hit rate since one large, popular object will prevent many
+ smaller, slightly less popular objects from being cached.
+
+ Both policies utilize a dynamic aging mechanism that prevents
+ cache pollution that can otherwise occur with frequency-based
+ replacement policies.
+
+ NOTE: if using the LFUDA replacement policy you should increase
+ the value of maximum_object_size above its default of 4096 KB to
+ to maximize the potential byte hit rate improvement of LFUDA.
+
+ For more information about the GDSF and LFUDA cache replacement
+ policies see http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html
+ and http://fog.hpl.external.hp.com/techreports/98/HPL-98-173.html.
+DOC_END
+
+NAME: memory_replacement_policy
+TYPE: removalpolicy
+LOC: Config.memPolicy
+DEFAULT: lru
+DOC_START
+ The memory replacement policy parameter determines which
+ objects are purged from memory when memory space is needed.
+
+ See cache_replacement_policy for details.
+DOC_END
+
+
COMMENT_START
LOGFILE PATHNAMES AND CACHE DIRECTORIES
-----------------------------------------------------------------------------
used.
DOC_END
-
-NAME: replacement_policy
-TYPE: string
-LOC: Config.replPolicy
-DEFAULT: LFUDA
-IFDEF: HEAP_REPLACEMENT
-DOC_START
- The cache replacement policy parameter determines which
- objects are evicted (replaced) when disk space is needed.
- Squid used to have only a single replacement policy, LRU.
- But when built with -DHEAP_REPLACEMENT you can choose
- between two new, enhanced policies:
-
- GDSF: Greedy-Dual Size Frequency
- LFUDA: Least Frequently Used with Dynamic Aging
-
- Both of these policies are frequency based rather than recency
- based, and perform better than LRU.
-
- The GDSF policy optimizes object hit rate by keeping smaller
- popular objects in cache so it has a better chance of getting a
- hit. It achieves a lower byte hit rate than LFUDA though since
- it evicts larger (possibly popular) objects.
-
- The LFUDA policy keeps popular objects in cache regardless of
- their size and thus optimizes byte hit rate at the expense of
- hit rate since one large, popular object will prevent many
- smaller, slightly less popular objects from being cached.
-
- Both policies utilize a dynamic aging mechanism that prevents
- cache pollution that can otherwise occur with frequency-based
- replacement policies.
-
- NOTE: if using the LFUDA replacement policy you should increase
- the value of maximum_object_size above its default of 4096 KB to
- to maximize the potential byte hit rate improvement of LFUDA.
-
- For more information about these cache replacement policies see
- http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html and
- http://fog.hpl.external.hp.com/techreports/98/HPL-98-173.html.
-DOC_END
-
-
NAME: reference_age
TYPE: time_t
LOC: Config.referenceAge
/*
- * $Id: async_io.cc,v 1.3 2000/05/29 21:06:28 hno Exp $
+ * $Id: async_io.cc,v 1.4 2000/06/08 18:05:37 hno Exp $
*
* DEBUG: section 32 Asynchronous Disk I/O
* AUTHOR: Pete Bentley <pete@demon.net>
cbdataLock(callback_data);
if (aio_unlink(path, &ctrlp->result) < 0) {
int ret = unlink(path);
- (callback) (ctrlp->fd, callback_data, ret, errno);
+ if (callback)
+ (callback) (ctrlp->fd, callback_data, ret, errno);
cbdataUnlock(callback_data);
memPoolFree(aio_ctrl_pool, ctrlp);
xfree(path);
} /* aioUnlink */
-void
+int
aioCheckCallbacks(SwapDir * SD)
{
aio_result_t *resultp;
aio_ctrl_t *prev;
AIOCB *done_handler;
void *their_data;
-
+ int retval = 0;
+
assert(initialised);
aio_counts.check_callback++;
for (;;) {
ctrlp->done_handler = NULL;
ctrlp->done_handler_data = NULL;
if (cbdataValid(their_data))
+ retval = 1; /* Return that we've actually done some work */
done_handler(ctrlp->fd, their_data,
ctrlp->result.aio_return, ctrlp->result.aio_errno);
cbdataUnlock(their_data);
aioFDWasClosed(ctrlp->fd);
memPoolFree(aio_ctrl_pool, ctrlp);
}
+ return retval;
}
void
void aioRead(int, int offset, char *, int size, AIOCB *, void *);
void aioStat(char *, struct stat *, AIOCB *, void *);
void aioUnlink(const char *, AIOCB *, void *);
-void aioCheckCallbacks(SwapDir *);
+int aioCheckCallbacks(SwapDir *);
void aioSync(SwapDir *);
int aioQueueSize(void);
/*
- * $Id: store_dir_aufs.cc,v 1.3 2000/05/29 21:06:28 hno Exp $
+ * $Id: store_dir_aufs.cc,v 1.4 2000/06/08 18:05:37 hno Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static STLOGOPEN storeAufsDirOpenSwapLog;
static STINIT storeAufsDirInit;
static STFREE storeAufsDirFree;
-static STLOGCLEANOPEN storeAufsDirWriteCleanOpen;
-static void storeAufsDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANSTART storeAufsDirWriteCleanStart;
+static STLOGCLEANNEXTENTRY storeAufsDirCleanLogNextEntry;
static STLOGCLEANWRITE storeAufsDirWriteCleanEntry;
+static STLOGCLEANDONE storeAufsDirWriteCleanDone;
static STLOGCLOSE storeAufsDirCloseSwapLog;
static STLOGWRITE storeAufsDirSwapLog;
static STNEWFS storeAufsDirNewfs;
static void storeAufsDirStats(SwapDir *, StoreEntry *);
static void storeAufsDirInitBitmap(SwapDir *);
static int storeAufsDirValidFileno(SwapDir *, sfileno, int);
-static int storeAufsDirCheckExpired(SwapDir *, StoreEntry *);
-#if !HEAP_REPLACEMENT
-static time_t storeAufsDirExpiredReferenceAge(SwapDir *);
-#endif
/*
* These functions were ripped straight out of the heart of store_dir.c.
{
static int started_clean_event = 0;
static const char *errmsg =
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.";
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
storeAufsDirInitBitmap(sd);
if (storeAufsDirVerifyCacheDirs(sd) < 0)
fatal(errmsg);
e->lastmod = s.lastmod;
e->flags = s.flags;
e->refcount += s.refcount;
-#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e, SD);
storeAufsDirUnrefObj(SD, e);
-#endif
} else {
debug_trap("storeAufsDirRebuildFromSwapLog: bad condition");
debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
e->swap_dirn = SD->index;
e->swap_file_sz = swap_file_sz;
e->lock_count = 0;
-#if !HEAP_REPLACEMENT
- e->refcount = 0;
-#endif
e->lastref = lastref;
e->timestamp = timestamp;
e->expires = expires;
char *outbuf;
off_t outbuf_offset;
int fd;
+ RemovalPolicyWalker *walker;
};
#define CLEAN_BUF_SZ 16384
* we succeed, and assign the 'func' and 'data' return pointers.
*/
static int
-storeAufsDirWriteCleanOpen(SwapDir * sd)
+storeAufsDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
struct stat sb;
state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
state->outbuf_offset = 0;
+ state->walker = sd->repl->WalkInit(sd->repl);
unlink(state->new);
unlink(state->cln);
state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
return 0;
}
+/*
+ * Get the next entry that is a candidate for clean log writing
+ */
+const StoreEntry *
+storeAufsDirCleanLogNextEntry(SwapDir * sd)
+{
+ const StoreEntry *entry = NULL;
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->walker)
+ entry = state->walker->Next(state->walker);
+ return entry;
+}
+
/*
* "write" an entry to the clean log file.
*/
static void
-storeAufsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+storeAufsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
{
storeSwapLogData s;
static size_t ss = sizeof(storeSwapLogData);
struct _clean_state *state = sd->log.clean.state;
- if (NULL == e) {
- storeAufsDirWriteCleanClose(sd);
- return;
- }
memset(&s, '\0', ss);
s.op = (char) SWAP_LOG_ADD;
s.swap_filen = e->swap_filen;
}
static void
-storeAufsDirWriteCleanClose(SwapDir * sd)
+storeAufsDirWriteCleanDone(SwapDir * sd)
{
struct _clean_state *state = sd->log.clean.state;
if (state->fd < 0)
return;
+ state->walker->Done(state->walker);
if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
state->new, xstrerror());
storeAufsDirMaintain(SwapDir * SD)
{
StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
+ int removed = 0;
int max_scan;
int max_remove;
double f;
- static time_t last_warn_time = 0;
-#if !HEAP_REPLACEMENT
- dlink_node *m;
- dlink_node *prev = NULL;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#if HEAP_REPLACEMENT_DEBUG
- if (!verify_heap_property(SD->repl.heap.heap)) {
- debug(20, 1) ("Heap property violated!\n");
- }
-#endif
-#endif
+ RemovalPurgeWalker *walker;
/* We can't delete objects while rebuilding swap */
if (store_dirs_rebuilding) {
return;
} else {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
max_scan = (int) (f * 400.0 + 100.0);
/*
* This is kinda cheap, but so we need this priority hack?
*/
-#if 0
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
-#endif
}
- debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
-#if HEAP_REPLACEMENT
- while (heap_nodes(SD->repl.heap.heap) > 0) {
- if (store_swap_size < store_swap_low)
- break;
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- age = heap_peepminkey(SD->repl.heap.heap);
- e = heap_extractmin(SD->repl.heap.heap);
- e->repl.node = NULL; /* no longer in the heap */
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * Entry is in use ... put it in a linked list to ignore it.
- */
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- /*
- * If this was a "SPECIAL" do not add it back into the heap.
- * It will always be "SPECIAL" and therefore never removed.
- */
- debug(20, 4) ("storeAufsDirMaintain: locked url %s\n",
- (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
- key));
- linklistPush(&locked_entries, e);
- }
- locked++;
- continue;
- } else if (storeAufsDirCheckExpired(SD, e)) {
- /*
- * Note: This will not check the reference age ifdef
- * HEAP_REPLACEMENT, but it does some other useful
- * checks...
- */
- expired++;
- debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
- age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
- min_age = age;
- storeRelease(e);
- } else {
- /*
- * Did not expire the object so we need to add it back
- * into the heap!
- */
- debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
+ debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n",
+ f, max_scan, max_remove);
+ walker = SD->repl->PurgeInit(SD->repl, max_scan);
+ while (1) {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
if (store_swap_size < store_swap_low)
break;
- else if (expired >= max_remove)
- break;
- else if (scanned >= max_scan)
+ if (removed >= max_remove)
break;
+ e = walker->Next(walker);
+ if (!e)
+ break; /* no more objects */
+ removed++;
+ storeRelease(e);
}
- /*
- * Bump the heap age factor.
- */
- if (min_age > 0.0)
- SD->repl.heap.heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
-#else
- for (m = SD->repl.lru.list.tail; m; m = prev) {
- prev = m->prev;
- e = m->data;
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
- locked++;
-
- } else if (storeAufsDirCheckExpired(SD, e)) {
- expired++;
- storeRelease(e);
- }
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- }
-#endif
- debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n",
- scanned, max_scan, expired, max_remove, locked, f);
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+ debug(20, (removed ? 2 : 3)) ("storeUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
+ SD->path, removed, max_remove, f, max_scan);
}
/*
int loadav;
int ql;
-#if !HEAP_REPLACEMENT
+#if OLD_UNUSED_CODE
if (storeAufsDirExpiredReferenceAge(SD) < 300) {
debug(20, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n",
storeAufsDirExpiredReferenceAge(SD));
{
debug(1, 3) ("storeAufsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
e->swap_filen);
-#if HEAP_REPLACEMENT
- /* Nothing to do here */
-#else
- /* Reference the object */
- if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
- !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
-#endif
+ if (SD->repl->Referenced)
+ SD->repl->Referenced(SD->repl, e, &e->repl);
}
/*
{
debug(1, 3) ("storeAufsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
e->swap_filen);
-#if HEAP_REPLACEMENT
- if (e->repl.node)
- heap_update(SD->repl.heap.heap, e->repl.node, e);
-#endif
+ if (SD->repl->Dereferenced)
+ SD->repl->Dereferenced(SD->repl, e, &e->repl);
}
/*
storeAufsDirUnlinkFile(SwapDir * SD, sfileno f)
{
debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f);
- storeAufsDirMapBitReset(SD, f);
+ /* storeAufsDirMapBitReset(SD, f); */
aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL);
}
-#if !HEAP_REPLACEMENT
-/*
- * storeAufsDirExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-static time_t
-storeAufsDirExpiredReferenceAge(SwapDir * SD)
-{
- double x;
- double z;
- time_t age;
- long store_high, store_low;
-
- store_high = (long) (((float) SD->max_size *
- (float) Config.Swap.highWaterMark) / (float) 100);
- store_low = (long) (((float) SD->max_size *
- (float) Config.Swap.lowWaterMark) / (float) 100);
- debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
-
- x = (double) (store_high - SD->cur_size) /
- (store_high - store_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > Config.referenceAge)
- age = Config.referenceAge;
- return age;
-}
-#endif
-
-/*
- * storeAufsDirCheckExpired
- *
- * Check whether the given object is expired or not
- * It breaks layering a little by calling the upper layers to find
- * out whether the object is locked or not, but we can't help this
- * right now.
- */
-static int
-storeAufsDirCheckExpired(SwapDir * SD, StoreEntry * e)
-{
- if (storeEntryLocked(e))
- return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
-
-#if HEAP_REPLACEMENT
- /*
- * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
- * controls the replacement of objects.
- */
- return 1;
-#else
- if (squid_curtime - e->lastref > storeAufsDirExpiredReferenceAge(SD))
- return 1;
- return 0;
-#endif
-}
-
/*
* Add and remove the given StoreEntry from the replacement policy in
* use.
{
debug(20, 4) ("storeAufsDirReplAdd: added node %p to dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- (void) 0;
- } else {
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
- debug(20, 4) ("storeAufsDirReplAdd: inserted node 0x%x\n", e->repl.node);
- }
-#else
- /* Shouldn't we not throw special objects into the lru ? */
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Add(SD->repl, e, &e->repl);
}
SwapDir *SD = INDEXSD(e->swap_dirn);
debug(20, 4) ("storeAufsDirReplRemove: remove node %p from dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- /* And now, release the object from the replacement policy */
- if (e->repl.node) {
- debug(20, 4) ("storeAufsDirReplRemove: deleting node 0x%x\n",
- e->repl.node);
- heap_delete(SD->repl.heap.heap, e->repl.node);
- e->repl.node = NULL;
- }
-#else
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Remove(SD->repl, e, &e->repl);
}
if (SD->flags.read_only)
storeAppendPrintf(sentry, " READ-ONLY");
storeAppendPrintf(sentry, "\n");
+#if OLD_UNUSED_CODE
#if !HEAP_REPLACEMENT
storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
(double) storeAufsDirExpiredReferenceAge(SD) / 86400.0);
#else
-#if 0
storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
heap_peepminkey(sd.repl.heap.heap));
#endif
-#endif
+#endif /* OLD_UNUSED_CODE */
}
/*
filemapFreeMemory(aioinfo->map);
xfree(aioinfo);
s->fsdata = NULL; /* Will aid debugging... */
-
}
char *
sd->log.open = storeAufsDirOpenSwapLog;
sd->log.close = storeAufsDirCloseSwapLog;
sd->log.write = storeAufsDirSwapLog;
- sd->log.clean.open = storeAufsDirWriteCleanOpen;
+ sd->log.clean.start = storeAufsDirWriteCleanStart;
+ sd->log.clean.nextentry = storeAufsDirCleanLogNextEntry;
+ sd->log.clean.done = storeAufsDirWriteCleanDone;
/* Initialise replacement policy stuff */
-#if HEAP_REPLACEMENT
- /*
- * Create new heaps with cache replacement policies attached to them.
- * The cache replacement policy is specified as either GDSF or LFUDA in
- * the squid.conf configuration file. Note that the replacement policy
- * applies only to the disk replacement algorithm. Memory replacement
- * always uses GDSF since we want to maximize object hit rate.
- */
- if (Config.replPolicy) {
- if (tolower(Config.replPolicy[0]) == 'g') {
- debug(20, 1) ("Using GDSF disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- } else if (tolower(Config.replPolicy[0]) == 'l') {
- if (tolower(Config.replPolicy[1]) == 'f') {
- debug(20, 1) ("Using LFUDA disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
- } else if (tolower(Config.replPolicy[1]) == 'r') {
- debug(20, 1) ("Using LRU heap disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
- }
- } else {
- debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
- } else {
- debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
-#else
- sd->repl.lru.list.head = NULL;
- sd->repl.lru.list.tail = NULL;
-#endif
+ sd->repl = createRemovalPolicy(Config.replPolicy);
}
/*
{
debug(78, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen);
storeAufsDirReplRemove(e);
+ storeAufsDirMapBitReset(SD, e->swap_filen);
storeAufsDirUnlinkFile(SD, e->swap_filen);
}
int fd;
int swaplog_fd;
int numcollisions;
+ dlink_list index;
+ int count;
+ dlink_node *walk_current;
+};
+
+struct _cossindex {
+ /* Note: the dlink_node MUST be the first member of the structure.
+ * This member is later pointer typecasted to coss_index_node *.
+ */
+ dlink_node node;
};
typedef struct _cossmembuf CossMemBuf;
typedef struct _cossinfo CossInfo;
typedef struct _cossstate CossState;
+typedef struct _cossindex CossIndexNode;
/* Whether the coss system has been setup or not */
extern int coss_initialised;
extern MemPool *coss_membuf_pool;
extern MemPool *coss_state_pool;
-
+extern MemPool *coss_index_pool;
/*
* Store IO stuff
extern STOBJREAD storeCossRead;
extern STOBJWRITE storeCossWrite;
extern STOBJUNLINK storeCossUnlink;
+extern STSYNC storeCossSync;
extern off_t storeCossAllocate(SwapDir * SD, const StoreEntry * e, int which);
-extern void storeCossFree(StoreEntry * e);
-extern void storeCossMaintainSwapSpace(SwapDir * SD);
-extern void storeCossDirStats(SwapDir *, StoreEntry *);
-extern void storeCossDirDump(StoreEntry * entry, const char *name, SwapDir * s);
-extern void storeCossDirFree(SwapDir *);
-extern SwapDir *storeCossDirPick(void);
-
-void storeFsSetup_ufs(storefs_entry_t *);
-
-
+extern void storeCossAdd(SwapDir *, StoreEntry *);
+extern void storeCossRemove(SwapDir *, StoreEntry *);
+extern void storeCossStartMembuf(SwapDir *SD);
#endif
/*
- * $Id: store_dir_coss.cc,v 1.2 2000/05/12 00:29:19 wessels Exp $
+ * $Id: store_dir_coss.cc,v 1.3 2000/06/08 18:05:38 hno Exp $
*
* DEBUG: section 81 Store COSS Directory Routines
* AUTHOR: Eric Stern
#define STORE_META_BUFSZ 4096
-extern void storeCossFlushMemBufs(SwapDir * SD);
-
int n_coss_dirs = 0;
/* static int last_coss_pick_index = -1; */
int coss_initialised = 0;
MemPool *coss_membuf_pool = NULL;
MemPool *coss_state_pool = NULL;
+MemPool *coss_index_pool = NULL;
typedef struct _RebuildState RebuildState;
struct _RebuildState {
static FILE *storeCossDirOpenTmpSwapLog(SwapDir *, int *, int *);
static STLOGOPEN storeCossDirOpenSwapLog;
static STINIT storeCossDirInit;
-static STLOGCLEANOPEN storeCossDirWriteCleanOpen;
-static void storeCossDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANSTART storeCossDirWriteCleanStart;
+static STLOGCLEANNEXTENTRY storeCossDirCleanLogNextEntry;
static STLOGCLEANWRITE storeCossDirWriteCleanEntry;
+static STLOGCLEANDONE storeCossDirWriteCleanDone;
static STLOGCLOSE storeCossDirCloseSwapLog;
static STLOGWRITE storeCossDirSwapLog;
static STNEWFS storeCossDirNewfs;
static STCHECKOBJ storeCossDirCheckObj;
-static void storeCossDirShutdown(SwapDir * sd);
-static void storeCossDirParse(SwapDir *, int, char *);
-static void storeCossDirReconfigure(SwapDir *, int, char *);
+static STFREE storeCossDirShutdown;
+static STFSPARSE storeCossDirParse;
+static STFSRECONFIGURE storeCossDirReconfigure;
+static STDUMP storeCossDirDump;
static char *
storeCossDirSwapLogFile(SwapDir * sd, const char *ext)
n_coss_dirs++;
}
+void
+storeCossRemove(SwapDir *sd, StoreEntry *e)
+{
+ CossInfo *cs = (CossInfo *) sd->fsdata;
+ CossIndexNode *coss_node = e->repl.data;
+ e->repl.data = NULL;
+ dlinkDelete(&coss_node->node, &cs->index);
+ memPoolFree(coss_index_pool, coss_node);
+ cs->count -= 1;
+}
+
+void
+storeCossAdd(SwapDir *sd, StoreEntry *e)
+{
+ CossInfo *cs = (CossInfo *) sd->fsdata;
+ CossIndexNode *coss_node = memPoolAlloc(coss_index_pool);
+ assert(!e->repl.data);
+ e->repl.data = coss_node;
+ dlinkAdd(e, &coss_node->node, &cs->index);
+ cs->count += 1;
+}
+
+static void
+storeCossRebuildComplete(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *sd = rb->sd;
+ storeCossStartMembuf(sd);
+ store_dirs_rebuilding--;
+ storeCossDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+}
+
static void
storeCossRebuildFromSwapLog(void *data)
{
RebuildState *rb = data;
+ SwapDir *sd = rb->sd;
StoreEntry *e = NULL;
storeSwapLogData s;
size_t ss = sizeof(storeSwapLogData);
rb->sd->path, rb->n_read);
fclose(rb->log);
rb->log = NULL;
- store_dirs_rebuilding--;
- storeCossDirCloseTmpSwapLog(rb->sd);
- storeRebuildComplete(&rb->counts);
- cbdataFree(rb);
+ storeCossRebuildComplete(rb);
return;
}
rb->n_read++;
}
storeRelease(e);
/* Fake an unlink here, this is a bad hack :( */
- dlinkDelete(&e->repl.lru, &rb->sd->repl.lru.list);
+ storeCossRemove(sd, e);
rb->counts.objcount--;
rb->counts.cancelcount++;
}
e->swap_filen = file_number;
e->swap_file_sz = swap_file_sz;
e->lock_count = 0;
-#if !HEAP_REPLACEMENT
- e->refcount = 0;
-#endif
e->lastref = lastref;
e->timestamp = timestamp;
e->expires = expires;
e->ping_status = PING_NONE;
EBIT_CLR(e->flags, ENTRY_VALIDATED);
storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ storeCossAdd(SD, e);
e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY);
return e;
}
int zero = 0;
FILE *fp;
EVH *func = NULL;
+ cbdataAdd(rb, cbdataXfree, 0);
rb->sd = sd;
rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ func = storeCossRebuildFromSwapLog;
+ rb->flags.clean = (unsigned int) clean;
/*
* If the swap.state file exists in the cache_dir, then
* we'll use storeCossRebuildFromSwapLog().
fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero);
debug(20, 1) ("Rebuilding COSS storage in %s (%s)\n",
sd->path, clean ? "CLEAN" : "DIRTY");
+ rb->log = fp;
+ store_dirs_rebuilding++;
if (!clean || fp == NULL) {
/* COSS cannot yet rebuild from a dirty state. If the log
* is dirty then the COSS contents is thrown away.
*/
if (fp != NULL)
fclose(fp);
- storeCossDirCloseTmpSwapLog(rb->sd);
/*
* XXX Make sure we don't trigger an assertion if this is the first
* storedir, since if we are, this call will cause storeRebuildComplete
* to prematurely complete the rebuild process, and then some other
* storedir will try to rebuild and eventually die.
*/
- store_dirs_rebuilding++;
- storeRebuildComplete(&rb->counts);
- store_dirs_rebuilding--;
- xfree(rb);
+ eventAdd("storeCossRebuildComplete", storeCossRebuildComplete, rb, 0.0, 0);
return;
}
- func = storeCossRebuildFromSwapLog;
- rb->log = fp;
- rb->flags.clean = (unsigned int) clean;
- store_dirs_rebuilding++;
- cbdataAdd(rb, cbdataXfree, 0);
eventAdd("storeCossRebuild", func, rb, 0.0, 1);
}
char *outbuf;
off_t outbuf_offset;
int fd;
+ dlink_node *current;
};
#define CLEAN_BUF_SZ 16384
* we succeed, and assign the 'func' and 'data' return pointers.
*/
static int
-storeCossDirWriteCleanOpen(SwapDir * sd)
+storeCossDirWriteCleanStart(SwapDir * sd)
{
+ CossInfo *cs = (CossInfo *) sd->fsdata;
struct _clean_state *state = xcalloc(1, sizeof(*state));
struct stat sb;
sd->log.clean.write = NULL;
state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
if (state->fd < 0)
return -1;
+ state->current = cs->index.tail;
debug(20, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n",
state->new, state->fd);
#if HAVE_FCHMOD
#endif
sd->log.clean.write = storeCossDirWriteCleanEntry;
sd->log.clean.state = state;
+
return 0;
}
+static const StoreEntry *
+storeCossDirCleanLogNextEntry(SwapDir * sd)
+{
+ struct _clean_state *state = sd->log.clean.state;
+ const StoreEntry *entry;
+ if (!state)
+ return NULL;
+ if (!state->current)
+ return NULL;
+ entry = (const StoreEntry *)state->current->data;
+ state->current = state->current->prev;
+ return entry;
+}
+
/*
* "write" an entry to the clean log file.
*/
static void
-storeCossDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+storeCossDirWriteCleanEntry(SwapDir *sd, const StoreEntry * e)
{
storeSwapLogData s;
static size_t ss = sizeof(storeSwapLogData);
struct _clean_state *state = sd->log.clean.state;
- if (NULL == e) {
- storeCossDirWriteCleanClose(sd);
- return;
- }
memset(&s, '\0', ss);
s.op = (char) SWAP_LOG_ADD;
s.swap_filen = e->swap_filen;
}
static void
-storeCossDirWriteCleanClose(SwapDir * sd)
+storeCossDirWriteCleanDone(SwapDir * sd)
{
struct _clean_state *state = sd->log.clean.state;
if (state->fd < 0)
storeCossDirShutdown(SwapDir * SD)
{
CossInfo *cs = (CossInfo *) SD->fsdata;
- CossMemBuf *t;
- /* we need to do this synchronously! */
- for (t = cs->membufs; t; t = t->next) {
- lseek(cs->fd, t->diskstart, SEEK_SET);
- write(cs->fd, t->buffer, COSS_MEMBUF_SZ);
- }
+ storeCossSync(SD);
+
file_close(cs->fd);
cs->fd = -1;
sd->refobj = NULL; /* LRU is done in storeCossRead */
sd->unrefobj = NULL;
sd->callback = NULL;
- sd->sync = NULL; /* should we make it call the coss sync? */
+ sd->sync = storeCossSync;
sd->obj.create = storeCossCreate;
sd->obj.open = storeCossOpen;
sd->log.open = storeCossDirOpenSwapLog;
sd->log.close = storeCossDirCloseSwapLog;
sd->log.write = storeCossDirSwapLog;
- sd->log.clean.open = storeCossDirWriteCleanOpen;
+ sd->log.clean.start = storeCossDirWriteCleanStart;
sd->log.clean.write = storeCossDirWriteCleanEntry;
+ sd->log.clean.nextentry = storeCossDirCleanLogNextEntry;
+ sd->log.clean.done = storeCossDirWriteCleanDone;
cs->current_offset = 0;
cs->fd = -1;
cs->swaplog_fd = -1;
cs->numcollisions = 0;
- cs->membufs = memPoolAlloc(coss_membuf_pool);
- cs->membufs->diskstart = 0;
- cs->membufs->diskend = COSS_MEMBUF_SZ;
- cs->membufs->lockcount = 0;
- cs->membufs->flags.full = 0;
- cs->membufs->flags.writing = 0;
- cs->membufs->next = NULL;
- cs->membufs->SD = sd;
+ cs->membufs = NULL; /* set when the rebuild completes */
cs->current_membuf = cs->membufs;
-
- sd->repl.lru.list.head = NULL;
- sd->repl.lru.list.tail = NULL;
+ cs->index.head = NULL;
+ cs->index.tail = NULL;
}
s->max_size >> 20);
}
-#if 0
+#if OLD_UNUSED_CODE
SwapDir *
storeCossDirPick(void)
{
storefs->donefunc = storeCossDirDone;
coss_membuf_pool = memPoolCreate("COSS Membuf data", sizeof(CossMemBuf));
coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState));
+ coss_index_pool = memPoolCreate("COSS index data", sizeof(CossIndexNode));
coss_initialised = 1;
}
/*
- * $Id: store_io_coss.cc,v 1.3 2000/05/16 07:09:36 wessels Exp $
+ * $Id: store_io_coss.cc,v 1.4 2000/06/08 18:05:38 hno Exp $
*
* DEBUG: section 81 Storage Manager COSS Interface
* AUTHOR: Eric Stern
else
checkf = -1;
- retofs = e->swap_filen; /* Just for defaults */
+ retofs = e->swap_filen; /* Just for defaults, or while rebuilding */
if (e->swap_file_sz > 0)
allocsize = e->swap_file_sz;
}
}
if (coll == 0) {
- cs->current_offset += allocsize;
+ cs->current_offset = retofs + allocsize;
return retofs;
} else {
return -1;
storeCossUnlink(SwapDir * SD, StoreEntry * e)
{
debug(81, 3) ("storeCossUnlink: offset %d\n", e->swap_filen);
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ storeCossRemove(SD, e);
}
sio->callback_data = callback_data;
cbdataLock(callback_data);
sio->e = (StoreEntry *) e;
- sio->st_size = -1; /* we won't know this until we read the metadata */
cstate->flags.writing = 0;
cstate->flags.reading = 0;
cstate->readbuffer = NULL;
cstate->reqdiskoffset = -1;
- /* Now add it into the LRU */
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ /* Now add it into the index list */
+ storeCossAdd(SD, e);
storeCossMemBufLock(SD, sio);
return sio;
storeCossMemBufLock(SD, sio);
/*
- * Do the LRU magic to keep the disk and memory LRUs identical
+ * Do the index magic to keep the disk and memory LRUs identical
*/
- dlinkDelete(&sio->e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(sio->e, &sio->e->repl.lru, &SD->repl.lru.list);
+ storeCossRemove(SD, e);
+ storeCossAdd(SD, e);
/*
* Since we've reallocated a spot for this object, we need to
}
}
+void
+storeCossSync(SwapDir * SD)
+{
+ CossInfo *cs = (CossInfo *) SD->fsdata;
+ CossMemBuf *t;
+ int end;
+ if (!cs->membufs)
+ return;
+ for (t=cs->membufs; t; t = t->next) {
+ if (t->flags.writing)
+ sleep(5);
+ lseek(cs->fd, t->diskstart, SEEK_SET);
+ end = (t == cs->current_membuf) ? cs->current_offset : t->diskend;
+ write(cs->fd, t->buffer, end - t->diskstart);
+ }
+}
static void
storeCossWriteMemBuf(SwapDir * SD, CossMemBuf * t)
debug(81, 3) ("storeCossWriteMemBuf: offset %d, len %d\n",
t->diskstart, t->diskend - t->diskstart);
cbdataAdd(t, storeCossMembufFree, 0);
+ t->flags.writing = 1;
file_write(cs->fd, t->diskstart, &t->buffer,
t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
- t->flags.writing = 1;
}
newmb->flags.writing = 0;
newmb->lockcount = 0;
newmb->SD = SD;
+ /* XXX This should be reversed, with the new buffer last in the chain */
newmb->next = cs->membufs;
cs->membufs = newmb;
for (t = cs->membufs; t; t = t->next)
debug(81, 3) ("storeCossCreateMemBuf: membuflist %d lockcount %d\n", t->diskstart, t->lockcount);
/*
- * XXX more evil LRU specific code. This needs to be replaced.
+ * Kill objects from the tail to make space for a new chunk
*/
- for (m = SD->repl.lru.list.tail; m; m = prev) {
+ for (m = cs->index.tail; m; m = prev) {
prev = m->prev;
e = m->data;
if (curfn == e->swap_filen)
return newmb;
}
+/*
+ * Creates the initial membuf after rebuild
+ */
+void
+storeCossStartMembuf(SwapDir *sd)
+{
+ CossInfo *cs = (CossInfo *) sd->fsdata;
+ CossMemBuf *newmb = storeCossCreateMemBuf(sd, cs->current_offset, -1, NULL);
+ assert(!cs->current_membuf);
+ cs->current_membuf = newmb;
+}
+
/*
* We can't pass memFree() as a free function here, because we need to free
* the fsstate variable ..
/*
- * $Id: store_dir_diskd.cc,v 1.10 2000/05/29 03:10:39 wessels Exp $
+ * $Id: store_dir_diskd.cc,v 1.11 2000/06/08 18:05:38 hno Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static STLOGOPEN storeDiskdDirOpenSwapLog;
static STINIT storeDiskdDirInit;
static STFREE storeDiskdDirFree;
-static STLOGCLEANOPEN storeDiskdDirWriteCleanOpen;
-static void storeDiskdDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANSTART storeDiskdDirWriteCleanStart;
+static STLOGCLEANNEXTENTRY storeDiskdDirCleanLogNextEntry;
static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry;
+static STLOGCLEANDONE storeDiskdDirWriteCleanDone;
static STLOGCLOSE storeDiskdDirCloseSwapLog;
static STLOGWRITE storeDiskdDirSwapLog;
static STNEWFS storeDiskdDirNewfs;
static void storeDiskdDirStats(SwapDir *, StoreEntry *);
static void storeDiskdDirInitBitmap(SwapDir *);
static int storeDiskdDirValidFileno(SwapDir *, sfileno, int);
-static int storeDiskdDirCheckExpired(SwapDir *, StoreEntry *);
-#if !HEAP_REPLACEMENT
-static time_t storeDiskdDirExpiredReferenceAge(SwapDir *);
-#endif
static void storeDiskdStats(StoreEntry * sentry);
static void storeDiskdDirSync(SwapDir *);
* until the queue is below magic2. Otherwise, we simply return when we
* don't get a message.
*/
-void
+int
storeDiskdDirCallback(SwapDir * SD)
{
diomsg M;
int x;
diskdinfo_t *diskdinfo = SD->fsdata;
+ int retval = 0;
- if (diskdinfo->away >= diskdinfo->magic2)
+ if (diskdinfo->away >= diskdinfo->magic2) {
diskd_stats.block_queue_len++;
+ retval = 1; /* We might not have anything to do, but our queue
+ * is full.. */
+ }
if (diskd_stats.sent_count - diskd_stats.recv_count >
diskd_stats.max_away) {
diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count;
diskd_stats.max_shmuse = diskd_stats.shmbuf_count;
}
- /* if we are above magic2, we do not break under any reason */
while (1) {
memset(&M, '\0', sizeof(M));
x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT);
- if (x < 0) {
- if (diskdinfo->away >= diskdinfo->magic2)
- continue;
- else
+ if (x < 0)
break;
- } else if (x != msg_snd_rcv_sz) {
+ else if (x != msg_snd_rcv_sz) {
debug(81, 1) ("storeDiskdDirCallback: msgget returns %d\n",
x);
break;
diskd_stats.recv_count++;
diskdinfo->away--;
storeDiskdHandle(&M);
+ retval = 1; /* Return that we've actually done some work */
if (M.shm_offset > -1)
storeDiskdShmPut(SD, M.shm_offset);
}
+ return retval;
}
e->lastmod = s.lastmod;
e->flags = s.flags;
e->refcount += s.refcount;
-#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e, SD);
storeDiskdDirUnrefObj(SD, e);
-#endif
} else {
debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition");
debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
e->swap_dirn = SD->index;
e->swap_file_sz = swap_file_sz;
e->lock_count = 0;
-#if !HEAP_REPLACEMENT
- e->refcount = 0;
-#endif
e->lastref = lastref;
e->timestamp = timestamp;
e->expires = expires;
char *outbuf;
off_t outbuf_offset;
int fd;
+ RemovalPolicyWalker *walker;
};
#define CLEAN_BUF_SZ 16384
* we succeed, and assign the 'func' and 'data' return pointers.
*/
static int
-storeDiskdDirWriteCleanOpen(SwapDir * sd)
+storeDiskdDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
struct stat sb;
state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean"));
state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
state->outbuf_offset = 0;
+ state->walker = sd->repl->WalkInit(sd->repl);
unlink(state->new);
unlink(state->cln);
state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
return 0;
}
+/*
+ * Get the next entry that is a candidate for clean log writing
+ */
+const StoreEntry *
+storeDiskdDirCleanLogNextEntry(SwapDir * sd)
+{
+ const StoreEntry *entry = NULL;
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->walker)
+ entry = state->walker->Next(state->walker);
+ return entry;
+}
+
/*
* "write" an entry to the clean log file.
*/
static void
-storeDiskdDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+storeDiskdDirWriteCleanEntry(SwapDir *sd, const StoreEntry * e)
{
storeSwapLogData s;
static size_t ss = sizeof(storeSwapLogData);
struct _clean_state *state = sd->log.clean.state;
- if (NULL == e) {
- storeDiskdDirWriteCleanClose(sd);
- return;
- }
memset(&s, '\0', ss);
s.op = (char) SWAP_LOG_ADD;
s.swap_filen = e->swap_filen;
}
static void
-storeDiskdDirWriteCleanClose(SwapDir * sd)
+storeDiskdDirWriteCleanDone(SwapDir * sd)
{
struct _clean_state *state = sd->log.clean.state;
if (state->fd < 0)
return;
+ state->walker->Done(state->walker);
if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
state->new, xstrerror());
storeDiskdDirMaintain(SwapDir * SD)
{
StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
+ int removed = 0;
int max_scan;
int max_remove;
double f;
- static time_t last_warn_time = 0;
-#if !HEAP_REPLACEMENT
- dlink_node *m;
- dlink_node *prev = NULL;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#if HEAP_REPLACEMENT_DEBUG
- if (!verify_heap_property(SD->repl.heap.heap)) {
- debug(20, 1) ("Heap property violated!\n");
- }
-#endif
-#endif
+ RemovalPurgeWalker *walker;
/* We can't delete objects while rebuilding swap */
if (store_dirs_rebuilding) {
return;
} else {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
max_scan = (int) (f * 400.0 + 100.0);
/*
* This is kinda cheap, but so we need this priority hack?
*/
-#if 0
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
-#endif
}
debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
-#if HEAP_REPLACEMENT
- while (heap_nodes(SD->repl.heap.heap) > 0) {
- if (store_swap_size < store_swap_low)
- break;
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- age = heap_peepminkey(SD->repl.heap.heap);
- e = heap_extractmin(SD->repl.heap.heap);
- e->repl.node = NULL; /* no longer in the heap */
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * Entry is in use ... put it in a linked list to ignore it.
- */
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- /*
- * If this was a "SPECIAL" do not add it back into the heap.
- * It will always be "SPECIAL" and therefore never removed.
- */
- debug(20, 4) ("storeDiskdDirMaintain: locked url %s\n",
- (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
- key));
- linklistPush(&locked_entries, e);
- }
- locked++;
- continue;
- } else if (storeDiskdDirCheckExpired(SD, e)) {
- /*
- * Note: This will not check the reference age ifdef
- * HEAP_REPLACEMENT, but it does some other useful
- * checks...
- */
- expired++;
- debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
- age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
- min_age = age;
- storeRelease(e);
- } else {
- /*
- * Did not expire the object so we need to add it back
- * into the heap!
- */
- debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
+ walker = SD->repl->PurgeInit(SD->repl, max_scan);
+ while (1) {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
if (store_swap_size < store_swap_low)
break;
- else if (expired >= max_remove)
- break;
- else if (scanned >= max_scan)
+ if (removed >= max_remove)
break;
+ e = walker->Next(walker);
+ if (!e)
+ break; /* no more objects */
+ removed++;
+ storeRelease(e);
}
- /*
- * Bump the heap age factor.
- */
- if (min_age > 0.0)
- SD->repl.heap.heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
-#else
- for (m = SD->repl.lru.list.tail; m; m = prev) {
- prev = m->prev;
- e = m->data;
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
- locked++;
-
- } else if (storeDiskdDirCheckExpired(SD, e)) {
- expired++;
- storeRelease(e);
- }
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- }
-#endif
- debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n",
- scanned, max_scan, expired, max_remove, locked, f);
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+ walker->Done(walker);
+ debug(20, (removed ? 2 : 3)) ("storeDiskdDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
+ SD->path, removed, max_remove, f, max_scan);
}
/*
int loadav;
diskdinfo_t *diskdinfo = SD->fsdata;
-#if !HEAP_REPLACEMENT
+#if OLD_UNUSED_CODE
if (storeDiskdDirExpiredReferenceAge(SD) < 300) {
debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n",
storeDiskdDirExpiredReferenceAge(SD));
{
debug(1, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
e->swap_filen);
-#if HEAP_REPLACEMENT
- /* Nothing to do here */
-#else
- /* Reference the object */
- if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
- !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
-#endif
+ if (SD->repl->Referenced)
+ SD->repl->Referenced(SD->repl, e, &e->repl);
}
/*
{
debug(1, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e,
e->swap_dirn, e->swap_filen);
-#if HEAP_REPLACEMENT
- if (e->repl.node)
- heap_update(SD->repl.heap.heap, e->repl.node, e);
-#endif
+ if (SD->repl->Dereferenced)
+ SD->repl->Dereferenced(SD->repl, e, &e->repl);
}
/*
storeDiskdDirUnlinkFile(SwapDir * SD, sfileno f)
{
debug(79, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f);
- storeDiskdDirMapBitReset(SD, f);
+ /* storeDiskdDirMapBitReset(SD, f); */
unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL));
}
-#if !HEAP_REPLACEMENT
-/*
- * storeDiskdDirExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-static time_t
-storeDiskdDirExpiredReferenceAge(SwapDir * SD)
-{
- double x;
- double z;
- time_t age;
- long store_high, store_low;
-
- store_high = (long) (((float) SD->max_size *
- (float) Config.Swap.highWaterMark) / (float) 100);
- store_low = (long) (((float) SD->max_size *
- (float) Config.Swap.lowWaterMark) / (float) 100);
- debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
-
- x = (double) (store_high - SD->cur_size) /
- (store_high - store_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > Config.referenceAge)
- age = Config.referenceAge;
- return age;
-}
-#endif
-
-/*
- * storeDiskdDirCheckExpired
- *
- * Check whether the given object is expired or not
- * It breaks layering a little by calling the upper layers to find
- * out whether the object is locked or not, but we can't help this
- * right now.
- */
-static int
-storeDiskdDirCheckExpired(SwapDir * SD, StoreEntry * e)
-{
- if (storeEntryLocked(e))
- return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
-
-#if HEAP_REPLACEMENT
- /*
- * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
- * controls the replacement of objects.
- */
- return 1;
-#else
- if (squid_curtime - e->lastref > storeDiskdDirExpiredReferenceAge(SD))
- return 1;
- return 0;
-#endif
-}
-
/*
* Add and remove the given StoreEntry from the replacement policy in
* use.
{
debug(20, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- (void) 0;
- } else {
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
- debug(20, 4) ("storeDiskdDirReplAdd: inserted node 0x%x\n", e->repl.node);
- }
-#else
- /* Shouldn't we not throw special objects into the lru ? */
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Add(SD->repl, e, &e->repl);
}
SwapDir *SD = INDEXSD(e->swap_dirn);
debug(20, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- /* And now, release the object from the replacement policy */
- if (e->repl.node) {
- debug(20, 4) ("storeDiskdDirReplRemove: deleting node 0x%x\n",
- e->repl.node);
- heap_delete(SD->repl.heap.heap, e->repl.node);
- e->repl.node = NULL;
- }
-#else
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Remove(SD->repl, e, &e->repl);
}
if (SD->flags.read_only)
storeAppendPrintf(sentry, " READ-ONLY");
storeAppendPrintf(sentry, "\n");
+#if OLD_UNUSED_CODE
#if !HEAP_REPLACEMENT
storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
(double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0);
#else
-#if 0
storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
heap_peepminkey(sd.repl.heap.heap));
#endif
-#endif
+#endif /* OLD_UNUSED_CODE */
storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away);
}
sd->log.open = storeDiskdDirOpenSwapLog;
sd->log.close = storeDiskdDirCloseSwapLog;
sd->log.write = storeDiskdDirSwapLog;
- sd->log.clean.open = storeDiskdDirWriteCleanOpen;
+ sd->log.clean.start = storeDiskdDirWriteCleanStart;
+ sd->log.clean.nextentry = storeDiskdDirCleanLogNextEntry;
+ sd->log.clean.done = storeDiskdDirWriteCleanDone;
/* Initialise replacement policy stuff */
-#if HEAP_REPLACEMENT
- /*
- * Create new heaps with cache replacement policies attached to them.
- * The cache replacement policy is specified as either GDSF or LFUDA in
- * the squid.conf configuration file. Note that the replacement policy
- * applies only to the disk replacement algorithm. Memory replacement
- * always uses GDSF since we want to maximize object hit rate.
- */
- if (Config.replPolicy) {
- if (tolower(Config.replPolicy[0]) == 'g') {
- debug(20, 1) ("Using GDSF disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- } else if (tolower(Config.replPolicy[0]) == 'l') {
- if (tolower(Config.replPolicy[1]) == 'f') {
- debug(20, 1) ("Using LFUDA disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
- } else if (tolower(Config.replPolicy[1]) == 'r') {
- debug(20, 1) ("Using LRU heap disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
- }
- } else {
- debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
- } else {
- debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
-#else
- sd->repl.lru.list.head = NULL;
- sd->repl.lru.list.tail = NULL;
-#endif
+ sd->repl = createRemovalPolicy(Config.replPolicy);
}
/*
extern void storeDiskdShmPut(SwapDir *, int);
extern void *storeDiskdShmGet(SwapDir *, int *);
extern void storeDiskdHandle(diomsg * M);
-extern void storeDiskdDirCallback(SwapDir *);
+extern int storeDiskdDirCallback(SwapDir *);
/*
/*
- * $Id: store_dir_ufs.cc,v 1.4 2000/05/29 03:10:42 wessels Exp $
+ * $Id: store_dir_ufs.cc,v 1.5 2000/06/08 18:05:39 hno Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static STLOGOPEN storeUfsDirOpenSwapLog;
static STINIT storeUfsDirInit;
static STFREE storeUfsDirFree;
-static STLOGCLEANOPEN storeUfsDirWriteCleanOpen;
-static void storeUfsDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANSTART storeUfsDirWriteCleanStart;
+static STLOGCLEANNEXTENTRY storeUfsDirCleanLogNextEntry;
static STLOGCLEANWRITE storeUfsDirWriteCleanEntry;
+static STLOGCLEANDONE storeUfsDirWriteCleanDone;
static STLOGCLOSE storeUfsDirCloseSwapLog;
static STLOGWRITE storeUfsDirSwapLog;
static STNEWFS storeUfsDirNewfs;
static void storeUfsDirStats(SwapDir *, StoreEntry *);
static void storeUfsDirInitBitmap(SwapDir *);
static int storeUfsDirValidFileno(SwapDir *, sfileno, int);
-static int storeUfsDirCheckExpired(SwapDir *, StoreEntry *);
-#if !HEAP_REPLACEMENT
-static time_t storeUfsDirExpiredReferenceAge(SwapDir *);
-#endif
/*
* These functions were ripped straight out of the heart of store_dir.c.
{
static int started_clean_event = 0;
static const char *errmsg =
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.";
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
storeUfsDirInitBitmap(sd);
if (storeUfsDirVerifyCacheDirs(sd) < 0)
fatal(errmsg);
e->lastmod = s.lastmod;
e->flags = s.flags;
e->refcount += s.refcount;
-#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e, SD);
storeUfsDirUnrefObj(SD, e);
-#endif
} else {
debug_trap("storeUfsDirRebuildFromSwapLog: bad condition");
debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
debug(20, 3) ("storeUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
rb->flags.init,
rb->sd->index,
- rb->curlvl1,
- rb->curlvl2);
+ rb->curlvl1, rb->curlvl2);
if (rb->done)
return -2;
while (fd < 0 && rb->done == 0) {
if (0 == rb->in_dir) { /* we need to read in a new directory */
snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
rb->sd->path,
- rb->curlvl1, rb->curlvl2);
+ rb->curlvl1,
+ rb->curlvl2);
if (rb->flags.init && rb->td != NULL)
closedir(rb->td);
rb->td = NULL;
e->swap_dirn = SD->index;
e->swap_file_sz = swap_file_sz;
e->lock_count = 0;
-#if !HEAP_REPLACEMENT
- e->refcount = 0;
-#endif
e->lastref = lastref;
e->timestamp = timestamp;
e->expires = expires;
char *outbuf;
off_t outbuf_offset;
int fd;
+ RemovalPolicyWalker *walker;
};
#define CLEAN_BUF_SZ 16384
* we succeed, and assign the 'func' and 'data' return pointers.
*/
static int
-storeUfsDirWriteCleanOpen(SwapDir * sd)
+storeUfsDirWriteCleanStart(SwapDir * sd)
{
struct _clean_state *state = xcalloc(1, sizeof(*state));
struct stat sb;
state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
state->outbuf_offset = 0;
+ state->walker = sd->repl->WalkInit(sd->repl);
unlink(state->new);
unlink(state->cln);
state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
return 0;
}
+/*
+ * Get the next entry that is a candidate for clean log writing
+ */
+const StoreEntry *
+storeUfsDirCleanLogNextEntry(SwapDir * sd)
+{
+ const StoreEntry *entry = NULL;
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->walker)
+ entry = state->walker->Next(state->walker);
+ return entry;
+}
+
/*
* "write" an entry to the clean log file.
*/
static void
-storeUfsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+storeUfsDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e)
{
storeSwapLogData s;
static size_t ss = sizeof(storeSwapLogData);
struct _clean_state *state = sd->log.clean.state;
- if (NULL == e) {
- storeUfsDirWriteCleanClose(sd);
- return;
- }
memset(&s, '\0', ss);
s.op = (char) SWAP_LOG_ADD;
s.swap_filen = e->swap_filen;
}
static void
-storeUfsDirWriteCleanClose(SwapDir * sd)
+storeUfsDirWriteCleanDone(SwapDir * sd)
{
struct _clean_state *state = sd->log.clean.state;
if (state->fd < 0)
return;
+ state->walker->Done(state->walker);
if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
state->new, xstrerror());
storeUfsDirMaintain(SwapDir * SD)
{
StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
+ int removed = 0;
int max_scan;
int max_remove;
double f;
- static time_t last_warn_time = 0;
-#if !HEAP_REPLACEMENT
- dlink_node *m;
- dlink_node *prev = NULL;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#if HEAP_REPLACEMENT_DEBUG
- if (!verify_heap_property(SD->repl.heap.heap)) {
- debug(20, 1) ("Heap property violated!\n");
- }
-#endif
-#endif
+ RemovalPurgeWalker *walker;
/* We can't delete objects while rebuilding swap */
if (store_dirs_rebuilding) {
return;
} else {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
max_scan = (int) (f * 400.0 + 100.0);
/*
* This is kinda cheap, but so we need this priority hack?
*/
-#if 0
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
-#endif
}
debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
-#if HEAP_REPLACEMENT
- while (heap_nodes(SD->repl.heap.heap) > 0) {
- if (store_swap_size < store_swap_low)
- break;
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- age = heap_peepminkey(SD->repl.heap.heap);
- e = heap_extractmin(SD->repl.heap.heap);
- e->repl.node = NULL; /* no longer in the heap */
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * Entry is in use ... put it in a linked list to ignore it.
- */
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- /*
- * If this was a "SPECIAL" do not add it back into the heap.
- * It will always be "SPECIAL" and therefore never removed.
- */
- debug(20, 4) ("storeUfsDirMaintain: locked url %s\n",
- (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
- key));
- linklistPush(&locked_entries, e);
- }
- locked++;
- continue;
- } else if (storeUfsDirCheckExpired(SD, e)) {
- /*
- * Note: This will not check the reference age ifdef
- * HEAP_REPLACEMENT, but it does some other useful
- * checks...
- */
- expired++;
- debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
- age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
- min_age = age;
- storeRelease(e);
- } else {
- /*
- * Did not expire the object so we need to add it back
- * into the heap!
- */
- debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
+ walker = SD->repl->PurgeInit(SD->repl, max_scan);
+ while (1) {
+ /* XXX FixMe: This should use the cache_dir hig/low values, not the
+ * global ones
+ */
if (store_swap_size < store_swap_low)
break;
- else if (expired >= max_remove)
- break;
- else if (scanned >= max_scan)
+ if (removed >= max_remove)
break;
+ e = walker->Next(walker);
+ if (!e)
+ break; /* no more objects */
+ removed++;
+ storeRelease(e);
}
- /*
- * Bump the heap age factor.
- */
- if (min_age > 0.0)
- SD->repl.heap.heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
-#else
- for (m = SD->repl.lru.list.tail; m; m = prev) {
- prev = m->prev;
- e = m->data;
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
- locked++;
-
- } else if (storeUfsDirCheckExpired(SD, e)) {
- expired++;
- storeRelease(e);
- }
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- }
-#endif
- debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n",
- scanned, max_scan, expired, max_remove, locked, f);
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+ walker->Done(walker);
+ debug(20, (removed ? 2 : 3)) ("storeUfsDirMaintain: %s removed %d/%d f=%.03f max_scan=%d\n",
+ SD->path, removed, max_remove, f, max_scan);
}
/*
int
storeUfsDirCheckObj(SwapDir * SD, const StoreEntry * e)
{
-#if !HEAP_REPLACEMENT
+#if OLD_UNUSED_CODE
if (storeUfsDirExpiredReferenceAge(SD) < 300) {
debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n",
storeUfsDirExpiredReferenceAge(SD));
{
debug(1, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
e->swap_filen);
-#if HEAP_REPLACEMENT
- /* Nothing to do here */
-#else
- /* Reference the object */
- if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
- !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
- }
-#endif
+ if (SD->repl->Referenced)
+ SD->repl->Referenced(SD->repl, e, &e->repl);
}
/*
{
debug(1, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
e->swap_filen);
-#if HEAP_REPLACEMENT
- if (e->repl.node)
- heap_update(SD->repl.heap.heap, e->repl.node, e);
-#endif
+ if (SD->repl->Dereferenced)
+ SD->repl->Dereferenced(SD->repl, e, &e->repl);
}
/*
storeUfsDirUnlinkFile(SwapDir * SD, sfileno f)
{
debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f);
- storeUfsDirMapBitReset(SD, f);
+ /* storeUfsDirMapBitReset(SD, f); */
unlinkdUnlink(storeUfsDirFullPath(SD, f, NULL));
}
-#if !HEAP_REPLACEMENT
-/*
- * storeUfsDirExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-static time_t
-storeUfsDirExpiredReferenceAge(SwapDir * SD)
-{
- double x;
- double z;
- time_t age;
- long store_high, store_low;
-
- store_high = (long) (((float) SD->max_size *
- (float) Config.Swap.highWaterMark) / (float) 100);
- store_low = (long) (((float) SD->max_size *
- (float) Config.Swap.lowWaterMark) / (float) 100);
- debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
-
- x = (double) (store_high - SD->cur_size) /
- (store_high - store_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > Config.referenceAge)
- age = Config.referenceAge;
- return age;
-}
-#endif
-
-/*
- * storeUfsDirCheckExpired
- *
- * Check whether the given object is expired or not
- * It breaks layering a little by calling the upper layers to find
- * out whether the object is locked or not, but we can't help this
- * right now.
- */
-static int
-storeUfsDirCheckExpired(SwapDir * SD, StoreEntry * e)
-{
- if (storeEntryLocked(e))
- return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
-
-#if HEAP_REPLACEMENT
- /*
- * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
- * controls the replacement of objects.
- */
- return 1;
-#else
- if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD))
- return 1;
- return 0;
-#endif
-}
-
/*
* Add and remove the given StoreEntry from the replacement policy in
* use.
{
debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- (void) 0;
- } else {
- e->repl.node = heap_insert(SD->repl.heap.heap, e);
- debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node);
- }
-#else
- /* Shouldn't we not throw special objects into the lru ? */
- dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Add(SD->repl, e, &e->repl);
}
SwapDir *SD = INDEXSD(e->swap_dirn);
debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e,
SD->index);
-#if HEAP_REPLACEMENT
- /* And now, release the object from the replacement policy */
- if (e->repl.node) {
- debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n",
- e->repl.node);
- heap_delete(SD->repl.heap.heap, e->repl.node);
- e->repl.node = NULL;
- }
-#else
- dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
-#endif
+ SD->repl->Remove(SD->repl, e, &e->repl);
}
if (SD->flags.read_only)
storeAppendPrintf(sentry, " READ-ONLY");
storeAppendPrintf(sentry, "\n");
+#if OLD_UNUSED_CODE
#if !HEAP_REPLACEMENT
storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
(double) storeUfsDirExpiredReferenceAge(SD) / 86400.0);
#else
-#if 0
storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
heap_peepminkey(sd.repl.heap.heap));
#endif
-#endif
+#endif /* OLD_UNUSED_CODE */
}
/*
sd->log.open = storeUfsDirOpenSwapLog;
sd->log.close = storeUfsDirCloseSwapLog;
sd->log.write = storeUfsDirSwapLog;
- sd->log.clean.open = storeUfsDirWriteCleanOpen;
+ sd->log.clean.start = storeUfsDirWriteCleanStart;
+ sd->log.clean.nextentry = storeUfsDirCleanLogNextEntry;
+ sd->log.clean.done = storeUfsDirWriteCleanDone;
/* Initialise replacement policy stuff */
-#if HEAP_REPLACEMENT
- /*
- * Create new heaps with cache replacement policies attached to them.
- * The cache replacement policy is specified as either GDSF or LFUDA in
- * the squid.conf configuration file. Note that the replacement policy
- * applies only to the disk replacement algorithm. Memory replacement
- * always uses GDSF since we want to maximize object hit rate.
- */
- if (Config.replPolicy) {
- if (tolower(Config.replPolicy[0]) == 'g') {
- debug(20, 1) ("Using GDSF disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- } else if (tolower(Config.replPolicy[0]) == 'l') {
- if (tolower(Config.replPolicy[1]) == 'f') {
- debug(20, 1) ("Using LFUDA disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
- } else if (tolower(Config.replPolicy[1]) == 'r') {
- debug(20, 1) ("Using LRU heap disk replacement policy\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
- }
- } else {
- debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
- } else {
- debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
- sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
-#else
- sd->repl.lru.list.head = NULL;
- sd->repl.lru.list.tail = NULL;
-#endif
+ sd->repl = createRemovalPolicy(Config.replPolicy);
}
/*
/*
- * $Id: globals.h,v 1.91 2000/05/31 04:26:32 wessels Exp $
+ * $Id: globals.h,v 1.92 2000/06/08 18:05:35 hno Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
extern double current_dtime;
extern int store_hash_buckets; /* 0 */
extern hash_table *store_table; /* NULL */
-#if HEAP_REPLACEMENT
-extern heap *inmem_heap;
-#else
-#endif
extern dlink_list ClientActiveRequests;
extern const String StringNull; /* { 0, 0, NULL } */
extern const MemBuf MemBufNull; /* MemBufNULL */
extern int store_swap_high; /* 0 */
extern int store_pages_max; /* 0 */
extern ssize_t store_maxobjsize; /* -1 */
+extern RemovalPolicy *mem_policy;
/*
- * $Id: protos.h,v 1.369 2000/05/31 07:01:42 hno Exp $
+ * $Id: protos.h,v 1.370 2000/06/08 18:05:35 hno Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
extern void storeFsDone(void);
extern void storeFsAdd(char *, STSETUP *);
-/* store_heap_replacement.c */
-#ifdef HEAP_REPLACEMENT
-extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *, double);
-extern heap_key HeapKeyGen_StoreEntry_GDSF(void *, double);
-extern heap_key HeapKeyGen_StoreEntry_LRU(void *, double);
-#endif
-
/* store_modules.c */
extern void storeFsSetup(void);
extern void logfilePrintf(va_alist);
#endif
+/* Removal Policies */
+RemovalPolicy *
+createRemovalPolicy(RemovalPolicySettings *settings);
+
/*
* prototypes for system functions missing from system includes
*/
--- /dev/null
+# Makefile for storage modules in the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/06/08 18:05:39 hno Exp $
+#
+
+SUBDIRS = @REPL_POLICIES@
+
+all:
+ @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \
+ sh -c "cd $$dir && $(MAKE) all" || exit 1; \
+ done; \
+ if [ ! -f stamp ]; then \
+ touch stamp; \
+ fi
+
+clean:
+ -rm -f *.a stamp
+ -for dir in *; do \
+ if [ -f $$dir/Makefile ]; then \
+ sh -c "cd $$dir && $(MAKE) $@" || exit 1;\
+ fi; \
+ done
+
+distclean:
+ -rm -f *.a Makefile
+ -for dir in *; do \
+ if [ -f $$dir/Makefile ]; then \
+ sh -c "cd $$dir && $(MAKE) distclean"; \
+ fi; \
+ done
+
+.DEFAULT:
+ @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \
+ sh -c "cd $$dir && $(MAKE) $@" || exit 1; \
+ done
--- /dev/null
+#
+# Makefile for the UFS storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/06/08 18:05:40 hno Exp $
+#
+
+REPL = heap
+
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(REPL).a
+
+OBJS = \
+ store_repl_$(REPL).o \
+ store_heap_replacement.o
+
+all: $(OUT)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) $(CFLAGS) -c $<
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+install:
+
+tags:
+ ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
--- /dev/null
+
+/*
+ * $Id: store_heap_replacement.cc,v 1.1 2000/06/08 18:05:40 hno Exp $
+ *
+ * DEBUG: section 20 Storage Manager Heap-based replacement
+ * AUTHOR: John Dilley
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * the Regents of the University of California. Please see the
+ * COPYRIGHT file for full details. Squid incorporates software
+ * developed and/or copyrighted by other sources. Please see the
+ * CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+/*
+ * The code in this file is Copyrighted (C) 1999 by Hewlett Packard.
+ *
+ *
+ * For a description of these cache replacement policies see --
+ * http://www.hpl.hp.com/techreports/1999/HPL-1999-69.html
+ */
+
+#include "squid.h"
+#include "heap.h"
+
+/*
+ * Key generation function to implement the LFU-DA policy (Least
+ * Frequently Used with Dynamic Aging). Similar to classical LFU
+ * but with aging to handle turnover of the popular document set.
+ * Maximizes byte hit rate by keeping more currently popular objects
+ * in cache regardless of size. Achieves lower hit rate than GDS
+ * because there are more large objects in cache (so less room for
+ * smaller popular objects).
+ *
+ * This version implements a tie-breaker based upon recency
+ * (e->lastref): for objects that have the same reference count
+ * the most recent object wins (gets a higher key value).
+ *
+ * Note: this does not properly handle when the aging factor
+ * gets so huge that the added value is outside of the
+ * precision of double. However, Squid has to stay up
+ * for quite a extended period of time (number of requests)
+ * for this to become a problem. (estimation is 10^8 cache
+ * turnarounds)
+ */
+heap_key HeapKeyGen_StoreEntry_LFUDA(void *entry, double age)
+{
+ StoreEntry *e = entry;
+ heap_key key;
+ double tie;
+ if (e->lastref <= 0)
+ tie = 0.0;
+ else if (squid_curtime <= e->lastref)
+ tie = 0.0;
+ else
+ tie = 1.0 - exp((double) (e->lastref - squid_curtime) / 86400.0);
+ key = age + (double) e->refcount - tie;
+ debug(81, 3) ("HeapKeyGen_StoreEntry_LFUDA: %s refcnt=%ld lastref=%ld age=%f tie=%f -> %f\n",
+ storeKeyText(e->key), e->refcount, e->lastref, age, tie, key);
+ if (e->mem_obj && e->mem_obj->url)
+ debug(81, 3) ("HeapKeyGen_StoreEntry_LFUDA: url=%s\n",
+ e->mem_obj->url);
+ return (double)key;
+}
+
+
+/*
+ * Key generation function to implement the GDS-Frequency policy.
+ * Similar to Greedy Dual-Size Hits policy, but adds aging of
+ * documents to prevent pollution. Maximizes object hit rate by
+ * keeping more small, popular objects in cache. Achieves lower
+ * byte hit rate than LFUDA because there are fewer large objects
+ * in cache.
+ *
+ * This version implements a tie-breaker based upon recency
+ * (e->lastref): for objects that have the same reference count
+ * the most recent object wins (gets a higher key value).
+ *
+ * Note: this does not properly handle when the aging factor
+ * gets so huge that the added value is outside of the
+ * precision of double. However, Squid has to stay up
+ * for quite a extended period of time (number of requests)
+ * for this to become a problem. (estimation is 10^8 cache
+ * turnarounds)
+ */
+heap_key HeapKeyGen_StoreEntry_GDSF(void *entry, double age)
+{
+ StoreEntry *e = entry;
+ heap_key key;
+ double size = e->swap_file_sz ? (double) e->swap_file_sz : 1.0;
+ double tie = (e->lastref > 1) ? (1.0 / e->lastref) : 1.0;
+ key = age + ((double) e->refcount / size) - tie;
+ debug(81, 3) ("HeapKeyGen_StoreEntry_GDSF: %s size=%f refcnt=%ld lastref=%ld age=%f tie=%f -> %f\n",
+ storeKeyText(e->key), size, e->refcount, e->lastref, age, tie, key);
+ if (e->mem_obj && e->mem_obj->url)
+ debug(81, 3) ("HeapKeyGen_StoreEntry_GDSF: url=%s\n",
+ e->mem_obj->url);
+ return key;
+}
+
+/*
+ * Key generation function to implement the LRU policy. Normally
+ * one would not do this with a heap -- use the linked list instead.
+ * For testing and performance characterization it was useful.
+ * Don't use it unless you are trying to compare performance among
+ * heap-based replacement policies...
+ */
+heap_key HeapKeyGen_StoreEntry_LRU(void *entry, double age)
+{
+ StoreEntry *e = entry;
+ debug(81, 3) ("HeapKeyGen_StoreEntry_LRU: %s age=%f lastref=%f\n",
+ storeKeyText(e->key), age, (double)e->lastref);
+ if (e->mem_obj && e->mem_obj->url)
+ debug(81, 3) ("HeapKeyGen_StoreEntry_LRU: url=%s\n",
+ e->mem_obj->url);
+ return (heap_key) e->lastref;
+}
--- /dev/null
+extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *entry, double age);
+extern heap_key HeapKeyGen_StoreEntry_GDSF(void *entry, double age);
+extern heap_key HeapKeyGen_StoreEntry_LRU(void *entry, double age);
--- /dev/null
+
+/*
+ * $Id: store_repl_heap.cc,v 1.1 2000/06/08 18:05:40 hno Exp $
+ *
+ * DEBUG: section ? HEAP based removal policies
+ * AUTHOR: Henrik Nordstrom
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "heap.h"
+#include "store_heap_replacement.h"
+
+REMOVALPOLICYCREATE createRemovalPolicy_heap;
+
+static int nr_heap_policies = 0;
+
+typedef struct _HeapPolicyData HeapPolicyData;
+struct _HeapPolicyData {
+ RemovalPolicy *policy;
+ heap *heap;
+ heap_key_func *keyfunc;
+ int count;
+ int nwalkers;
+ enum heap_entry_type
+ { TYPE_UNKNOWN = 0, TYPE_STORE_ENTRY, TYPE_STORE_MEM }
+ type;
+};
+
+/* Hack to avoid having to remember the RemovalPolicyNode location.
+ * Needed by the purge walker.
+ */
+static enum heap_entry_type
+heap_guessType(StoreEntry * entry, RemovalPolicyNode * node)
+{
+ if (node == &entry->repl)
+ return TYPE_STORE_ENTRY;
+ if (entry->mem_obj && node == &entry->mem_obj->repl)
+ return TYPE_STORE_MEM;
+ fatal("Heap Replacement: Unknown StoreEntry node type");
+ return TYPE_UNKNOWN;
+}
+#define SET_POLICY_NODE(entry,value) \
+ switch(heap->type) { \
+ case TYPE_STORE_ENTRY: entry->repl.data = value; break ; \
+ case TYPE_STORE_MEM: entry->mem_obj->repl.data = value ; break ; \
+ default: break; \
+ }
+
+static void
+heap_add(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node)
+{
+ HeapPolicyData *heap = policy->_data;
+ assert(!node->data);
+ if (EBIT_TEST(entry->flags, ENTRY_SPECIAL))
+ return; /* We won't manage these.. they messes things up */
+ node->data = heap_insert(heap->heap, entry);
+ heap->count += 1;
+ if (!heap->type)
+ heap->type = heap_guessType(entry, node);
+ /* Add a little more variance to the aging factor */
+ heap->heap->age += heap->heap->age / 100000000;
+}
+
+static void
+heap_remove(RemovalPolicy * policy, StoreEntry * entry,
+ RemovalPolicyNode * node)
+{
+ HeapPolicyData *heap = policy->_data;
+ heap_node *hnode = node->data;
+ if (!hnode)
+ return;
+ heap_delete(heap->heap, hnode);
+ node->data = NULL;
+ heap->count -= 1;
+}
+
+static void
+heap_referenced(RemovalPolicy * policy, const StoreEntry * entry,
+ RemovalPolicyNode * node)
+{
+ HeapPolicyData *heap = policy->_data;
+ heap_node *hnode = node->data;
+ if (!hnode)
+ return;
+ heap_update(heap->heap, hnode, (StoreEntry *) entry);
+}
+
+/** RemovalPolicyWalker **/
+
+typedef struct _HeapWalkData HeapWalkData;
+struct _HeapWalkData
+{
+ int current;
+};
+
+const StoreEntry *
+heap_walkNext(RemovalPolicyWalker * walker)
+{
+ HeapWalkData *heap_walk = walker->_data;
+ RemovalPolicy *policy = walker->_policy;
+ HeapPolicyData *heap = policy->_data;
+ StoreEntry *entry;
+ if (heap_walk->current >= heap_nodes(heap->heap))
+ return NULL; /* done */
+ entry = (StoreEntry *) heap_peep(heap->heap, heap_walk->current++);
+ return entry;
+}
+
+static void
+heap_walkDone(RemovalPolicyWalker * walker)
+{
+ RemovalPolicy *policy = walker->_policy;
+ HeapPolicyData *heap = policy->_data;
+ assert(strcmp(policy->_type, "heap") == 0);
+ assert(heap->nwalkers > 0);
+ heap->nwalkers -= 1;
+ safe_free(walker->_data);
+ cbdataFree(walker);
+}
+
+static RemovalPolicyWalker *
+heap_walkInit(RemovalPolicy * policy)
+{
+ HeapPolicyData *heap = policy->_data;
+ RemovalPolicyWalker *walker;
+ HeapWalkData *heap_walk;
+ heap->nwalkers += 1;
+ walker = xcalloc(1, sizeof(*walker));
+ heap_walk = xcalloc(1, sizeof(*heap_walk));
+ heap_walk->current = 0;
+ walker->_policy = policy;
+ walker->_data = heap_walk;
+ walker->Next = heap_walkNext;
+ walker->Done = heap_walkDone;
+ cbdataAdd(walker, cbdataXfree, 0);
+ return walker;
+}
+
+/** RemovalPurgeWalker **/
+
+typedef struct _HeapPurgeData HeapPurgeData;
+struct _HeapPurgeData
+{
+ link_list *locked_entries;
+ heap_key min_age;
+};
+
+static StoreEntry *
+heap_purgeNext(RemovalPurgeWalker * walker)
+{
+ HeapPurgeData *heap_walker = walker->_data;
+ RemovalPolicy *policy = walker->_policy;
+ HeapPolicyData *heap = policy->_data;
+ StoreEntry *entry;
+ heap_key age;
+ try_again:
+ if (!heap_nodes(heap->heap) > 0)
+ return NULL; /* done */
+ age = heap_peepminkey(heap->heap);
+ entry = heap_extractmin(heap->heap);
+ if (storeEntryLocked(entry)) {
+ linklistPush(&heap_walker->locked_entries, entry);
+ goto try_again;
+ }
+ heap_walker->min_age = age;
+ SET_POLICY_NODE(entry, NULL);
+ return entry;
+}
+
+static void
+heap_purgeDone(RemovalPurgeWalker * walker)
+{
+ HeapPurgeData *heap_walker = walker->_data;
+ RemovalPolicy *policy = walker->_policy;
+ HeapPolicyData *heap = policy->_data;
+ StoreEntry *entry;
+ assert(strcmp(policy->_type, "heap") == 0);
+ assert(heap->nwalkers > 0);
+ heap->nwalkers -= 1;
+ if (heap_walker->min_age > 0) {
+ heap->heap->age = heap_walker->min_age;
+ debug (81, 3) ("heap_purgeDone: Heap age set to %f\n",
+ (double) heap->heap->age);
+ }
+ /*
+ * Reinsert the locked entries
+ */
+ while ((entry = linklistShift(&heap_walker->locked_entries))) {
+ heap_node *node = heap_insert(heap->heap, entry);
+ SET_POLICY_NODE(entry, node);
+ }
+ safe_free(walker->_data);
+ cbdataFree(walker);
+}
+
+static RemovalPurgeWalker *
+heap_purgeInit(RemovalPolicy * policy, int max_scan)
+{
+ HeapPolicyData *heap = policy->_data;
+ RemovalPurgeWalker *walker;
+ HeapPurgeData *heap_walk;
+ heap->nwalkers += 1;
+ walker = xcalloc(1, sizeof(*walker));
+ heap_walk = xcalloc(1, sizeof(*heap_walk));
+ heap_walk->min_age = 0.0;
+ heap_walk->locked_entries = NULL;
+ walker->_policy = policy;
+ walker->_data = heap_walk;
+ walker->max_scan = max_scan;
+ walker->Next = heap_purgeNext;
+ walker->Done = heap_purgeDone;
+ cbdataAdd(walker, cbdataXfree, 0);
+#if HEAP_REPLACEMENT_DEBUG
+ if (!verify_heap_property(heap->heap)) {
+ debug(81, 1) ("Heap property violated!\n");
+ }
+#endif
+ return walker;
+}
+
+static void
+heap_free(RemovalPolicy * policy)
+{
+ HeapPolicyData *heap = policy->_data;
+ /* Make some verification of the policy state */
+ assert(strcmp(policy->_type, "heap") == 0);
+ assert(heap->nwalkers);
+ assert(heap->count);
+ /* Ok, time to destroy this policy */
+ safe_free(policy->_data);
+ memset(policy, 0, sizeof(*policy));
+ cbdataFree(policy);
+}
+
+RemovalPolicy *
+createRemovalPolicy_heap(wordlist * args)
+{
+ RemovalPolicy *policy;
+ HeapPolicyData *heap_data;
+ char *keytype;
+ /* Allocate the needed structures */
+ policy = xcalloc(1, sizeof(*policy));
+ heap_data = xcalloc(1, sizeof(*heap_data));
+ /* cbdata register the policy */
+ cbdataAdd(policy, cbdataXfree, 0);
+ /* Initialize the policy data */
+ heap_data->policy = policy;
+ if (args) {
+ keytype = args->key;
+ args = args->next;
+ } else {
+ debug(81, 1) ("createRemovalPolicy_heap: No key type specified. Using LRU\n");
+ keytype = "LRU";
+ }
+ if (!strcmp(keytype, "GDSF"))
+ heap_data->keyfunc = HeapKeyGen_StoreEntry_GDSF;
+ else if (!strcmp(keytype, "LFUDA"))
+ heap_data->keyfunc = HeapKeyGen_StoreEntry_LFUDA;
+ else if (!strcmp(keytype, "LRU"))
+ heap_data->keyfunc = HeapKeyGen_StoreEntry_LRU;
+ else {
+ debug(81, 0) ("createRemovalPolicy_heap: Unknown key type \"%s\". Using LRU\n",
+ keytype);
+ heap_data->keyfunc = HeapKeyGen_StoreEntry_LRU;
+ }
+ /* No additional arguments expected */
+ assert(!args);
+ heap_data->heap = new_heap(1000, heap_data->keyfunc);
+ heap_data->heap->age = 1.0;
+ /* Populate the policy structure */
+ policy->_type = "heap";
+ policy->_data = heap_data;
+ policy->Free = heap_free;
+ policy->Add = heap_add;
+ policy->Remove = heap_remove;
+ policy->Referenced = NULL;
+ policy->Dereferenced = heap_referenced;
+ policy->WalkInit = heap_walkInit;
+ policy->PurgeInit = heap_purgeInit;
+ /* Increase policy usage count */
+ nr_heap_policies += 0;
+ return policy;
+}
--- /dev/null
+#
+# Makefile for the UFS storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/06/08 18:05:40 hno Exp $
+#
+
+REPL = lru
+
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(REPL).a
+
+OBJS = \
+ store_repl_$(REPL).o
+
+all: $(OUT)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) $(CFLAGS) -c $<
+
+install:
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+tags:
+ ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
--- /dev/null
+
+/*
+ * $Id: store_repl_lru.cc,v 1.1 2000/06/08 18:05:40 hno Exp $
+ *
+ * DEBUG: section ? LRU Removal policy
+ * AUTHOR: Henrik Nordstrom
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+REMOVALPOLICYCREATE createRemovalPolicy_lru;
+
+typedef struct _LruPolicyData LruPolicyData;
+struct _LruPolicyData {
+ RemovalPolicy *policy;
+ dlink_list list;
+ int count;
+ int nwalkers;
+ enum heap_entry_type
+ { TYPE_UNKNOWN = 0, TYPE_STORE_ENTRY, TYPE_STORE_MEM }
+ type;
+};
+
+/* Hack to avoid having to remember the RemovalPolicyNode location.
+ * Needed by the purge walker to clear the policy information
+ */
+static enum heap_entry_type
+repl_guessType(StoreEntry * entry, RemovalPolicyNode * node)
+{
+ if (node == &entry->repl)
+ return TYPE_STORE_ENTRY;
+ if (entry->mem_obj && node == &entry->mem_obj->repl)
+ return TYPE_STORE_MEM;
+ fatal("Heap Replacement: Unknown StoreEntry node type");
+ return TYPE_UNKNOWN;
+}
+#define SET_POLICY_NODE(entry,value) \
+ switch(lru->type) { \
+ case TYPE_STORE_ENTRY: entry->repl.data = value; break ; \
+ case TYPE_STORE_MEM: entry->mem_obj->repl.data = value ; break ; \
+ default: break; \
+ }
+
+typedef struct _LruNode LruNode;
+struct _LruNode
+{
+ /* Note: the dlink_node MUST be the first member of the LruNode
+ * structure. This member is later pointer typecasted to LruNode *.
+ */
+ dlink_node node;
+};
+
+static MemPool *lru_node_pool = NULL;
+static int nr_lru_policies = 0;
+
+static void
+lru_add(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node)
+{
+ LruPolicyData *lru = policy->_data;
+ LruNode *lru_node;
+ assert(!node->data);
+ node->data = lru_node = memPoolAlloc(lru_node_pool);
+ dlinkAddTail(entry, &lru_node->node, &lru->list);
+ lru->count += 1;
+ if (!lru->type)
+ lru->type = repl_guessType(entry, node);
+}
+
+static void
+lru_remove(RemovalPolicy * policy, StoreEntry * entry, RemovalPolicyNode * node)
+{
+ LruPolicyData *lru = policy->_data;
+ LruNode *lru_node = node->data;
+ if (!lru_node)
+ return;
+ assert(lru_node->node.data == entry);
+ node->data = NULL;
+ dlinkDelete(&lru_node->node, &lru->list);
+ memPoolFree(lru_node_pool, lru_node);
+ lru->count -= 1;
+}
+
+static void
+lru_referenced(RemovalPolicy * policy, const StoreEntry * entry,
+ RemovalPolicyNode * node)
+{
+ LruPolicyData *lru = policy->_data;
+ LruNode *lru_node = node->data;
+ if (!lru_node)
+ return;
+ dlinkDelete(&lru_node->node, &lru->list);
+ dlinkAddTail((void *) entry, &lru_node->node, &lru->list);
+}
+
+/** RemovalPolicyWalker **/
+
+typedef struct _LruWalkData LruWalkData;
+struct _LruWalkData
+{
+ LruNode *current;
+};
+
+const StoreEntry *
+lru_walkNext(RemovalPolicyWalker * walker)
+{
+ LruWalkData *lru_walk = walker->_data;
+ LruNode *lru_node = lru_walk->current;
+ if (!lru_node)
+ return NULL;
+ lru_walk->current = (LruNode *) lru_node->node.next;
+ return (StoreEntry *) lru_node->node.data;
+}
+
+static void
+lru_walkDone(RemovalPolicyWalker * walker)
+{
+ RemovalPolicy *policy = walker->_policy;
+ LruPolicyData *lru = policy->_data;
+ assert(strcmp(policy->_type, "lru") == 0);
+ assert(lru->nwalkers > 0);
+ lru->nwalkers -= 1;
+ safe_free(walker->_data);
+ cbdataFree(walker);
+}
+
+static RemovalPolicyWalker *
+lru_walkInit(RemovalPolicy * policy)
+{
+ LruPolicyData *lru = policy->_data;
+ RemovalPolicyWalker *walker;
+ LruWalkData *lru_walk;
+ lru->nwalkers += 1;
+ walker = xcalloc(1, sizeof(*walker));
+ lru_walk = xcalloc(1, sizeof(*lru_walk));
+ walker->_policy = policy;
+ walker->_data = lru_walk;
+ walker->Next = lru_walkNext;
+ walker->Done = lru_walkDone;
+ lru_walk->current = (LruNode *) lru->list.head;
+ cbdataAdd(walker, cbdataXfree, 0);
+ return walker;
+}
+
+/** RemovalPurgeWalker **/
+
+typedef struct _LruPurgeData LruPurgeData;
+struct _LruPurgeData
+{
+ LruNode *current;
+ LruNode *start;
+};
+
+static StoreEntry *
+lru_purgeNext(RemovalPurgeWalker * walker)
+{
+ LruPurgeData *lru_walker = walker->_data;
+ RemovalPolicy *policy = walker->_policy;
+ LruPolicyData *lru = policy->_data;
+ LruNode *lru_node;
+ StoreEntry *entry;
+ try_again:
+ lru_node = lru_walker->current;
+ if (!lru_node || walker->scanned >= walker->max_scan)
+ return NULL;
+ walker->scanned += 1;
+ lru_walker->current = (LruNode *) lru_node->node.next;
+ if (lru_walker->current == lru_walker->start) {
+ /* Last node found */
+ lru_walker->current = NULL;
+ }
+ entry = (StoreEntry *) lru_node->node.data;
+ dlinkDelete(&lru_node->node, &lru->list);
+ if (storeEntryLocked(entry)) {
+ /* Shit, it is locked. we can't return this one */
+ walker->locked++;
+ dlinkAddTail(entry, &lru_node->node, &lru->list);
+ goto try_again;
+ }
+ memPoolFree(lru_node_pool, lru_node);
+ lru->count -= 1;
+ SET_POLICY_NODE(entry, NULL);
+ return entry;
+}
+
+static void
+lru_purgeDone(RemovalPurgeWalker * walker)
+{
+ RemovalPolicy *policy = walker->_policy;
+ LruPolicyData *lru = policy->_data;
+ assert(strcmp(policy->_type, "lru") == 0);
+ assert(lru->nwalkers > 0);
+ lru->nwalkers -= 1;
+ safe_free(walker->_data);
+ cbdataFree(walker);
+}
+
+static RemovalPurgeWalker *
+lru_purgeInit(RemovalPolicy * policy, int max_scan)
+{
+ LruPolicyData *lru = policy->_data;
+ RemovalPurgeWalker *walker;
+ LruPurgeData *lru_walk;
+ lru->nwalkers += 1;
+ walker = xcalloc(1, sizeof(*walker));
+ lru_walk = xcalloc(1, sizeof(*lru_walk));
+ walker->_policy = policy;
+ walker->_data = lru_walk;
+ walker->max_scan = max_scan;
+ walker->Next = lru_purgeNext;
+ walker->Done = lru_purgeDone;
+ lru_walk->start = lru_walk->current = (LruNode *) lru->list.head;
+ cbdataAdd(walker, cbdataXfree, 0);
+ return walker;
+}
+
+static void
+lru_free(RemovalPolicy * policy)
+{
+ LruPolicyData *lru = policy->_data;
+ /* Make some verification of the policy state */
+ assert(strcmp(policy->_type, "lru") == 0);
+ assert(lru->nwalkers);
+ assert(lru->count);
+ /* Ok, time to destroy this policy */
+ safe_free(policy->_data);
+ memset(policy, 0, sizeof(*policy));
+ cbdataFree(policy);
+}
+
+RemovalPolicy *
+createRemovalPolicy_lru(wordlist *args)
+{
+ RemovalPolicy *policy;
+ LruPolicyData *lru_data;
+ /* no arguments expected or understood */
+ assert(!args);
+ /* Initialize */
+ if (!lru_node_pool)
+ lru_node_pool = memPoolCreate("LRU policy node", sizeof(LruNode));
+ /* Allocate the needed structures */
+ policy = xcalloc(1, sizeof(*policy));
+ lru_data = xcalloc(1, sizeof(*lru_data));
+ /* cbdata register the policy */
+ cbdataAdd(policy, cbdataXfree, 0);
+ /* Initialize the URL data */
+ lru_data->policy = policy;
+ /* Populate the policy structure */
+ policy->_type = "lru";
+ policy->_data = lru_data;
+ policy->Free = lru_free;
+ policy->Add = lru_add;
+ policy->Remove = lru_remove;
+ policy->Referenced = lru_referenced;
+ policy->Dereferenced = lru_referenced;
+ policy->WalkInit = lru_walkInit;
+ policy->PurgeInit = lru_purgeInit;
+ /* Increase policy usage count */
+ nr_lru_policies += 0;
+ return policy;
+}
+
+
+#if OLD_UNUSED_CODE
+/*
+ * storeUfsDirCheckExpired
+ *
+ * Check whether the given object is expired or not
+ * It breaks layering a little by calling the upper layers to find
+ * out whether the object is locked or not, but we can't help this
+ * right now.
+ */
+static int
+storeUfsDirCheckExpired(SwapDir * SD, StoreEntry * e)
+{
+ if (storeEntryLocked(e))
+ return 0;
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ return 1;
+ if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
+ return 1;
+
+#if HEAP_REPLACEMENT
+ /*
+ * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
+ * controls the replacement of objects.
+ */
+ return 1;
+#else
+ if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD))
+ return 1;
+ return 0;
+#endif
+}
+
+/*
+ * storeUfsDirExpiredReferenceAge
+ *
+ * The LRU age is scaled exponentially between 1 minute and
+ * Config.referenceAge , when store_swap_low < store_swap_size <
+ * store_swap_high. This keeps store_swap_size within the low and high
+ * water marks. If the cache is very busy then store_swap_size stays
+ * closer to the low water mark, if it is not busy, then it will stay
+ * near the high water mark. The LRU age value can be examined on the
+ * cachemgr 'info' page.
+ */
+static time_t
+storeUfsDirExpiredReferenceAge(SwapDir * SD)
+{
+ double x;
+ double z;
+ time_t age;
+ long store_high, store_low;
+
+ store_high = (long) (((float) SD->max_size *
+ (float) Config.Swap.highWaterMark) / (float) 100);
+ store_low = (long) (((float) SD->max_size *
+ (float) Config.Swap.lowWaterMark) / (float) 100);
+ debug(81, 4) ("storeUfsDirExpiredReferenceAge: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high,
+ store_low, SD->cur_size);
+
+ x = (double) (store_high - SD->cur_size) / (store_high - store_low);
+ x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
+ z = pow((double) (Config.referenceAge / 60), x);
+ age = (time_t) (z * 60.0);
+ if (age < 60)
+ age = 60;
+ else if (age > Config.referenceAge)
+ age = Config.referenceAge;
+ return age;
+}
+#endif /* OLD_UNUSED_CODE */
/*
- * $Id: squid.h,v 1.201 2000/05/03 17:15:42 adrian Exp $
+ * $Id: squid.h,v 1.202 2000/06/08 18:05:35 hno Exp $
*
* AUTHOR: Duane Wessels
*
#include "hash.h"
#include "rfc1035.h"
-#if HEAP_REPLACEMENT
-#include "heap.h"
-#endif
#include "defines.h"
#include "enums.h"
/*
- * $Id: store.cc,v 1.524 2000/05/28 17:00:13 wessels Exp $
+ * $Id: store.cc,v 1.525 2000/06/08 18:05:36 hno Exp $
*
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
static void destroy_MemObject(StoreEntry *);
static FREE destroy_StoreEntry;
static void storePurgeMem(StoreEntry *);
+static void storeEntryReferenced(StoreEntry *);
+static void storeEntryDereferenced(StoreEntry *);
static int getKeyCounter(void);
static int storeKeepInMemory(const StoreEntry *);
static OBJH storeCheckCachableStats;
/*
* local variables
*/
-#if HEAP_REPLACEMENT
-/*
- * The heap equivalent of inmem_list, inmem_heap, is in globals.c so other
- * modules can access it when updating object metadata (e.g., refcount)
- */
-#else
-static dlink_list inmem_list;
-#endif
static Stack LateReleaseStack;
#if URL_CHECKSUM_DEBUG
storeRelease(e);
}
-void
-storeLockObject(StoreEntry * e)
+static void
+storeEntryReferenced(StoreEntry *e)
{
SwapDir *SD;
- if (e->swap_dirn > -1)
+ /* Notify the fs that we're referencing this object again */
+ if (e->swap_dirn > -1) {
SD = INDEXSD(e->swap_dirn);
- else
- SD = NULL;
+ if (SD->refobj)
+ SD->refobj(SD, e);
+ }
+ /* Notify the memory cache that we're referencing this object again */
+ if(e->mem_obj) {
+ if (mem_policy->Referenced)
+ mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+static void
+storeEntryDereferenced(StoreEntry *e)
+{
+ SwapDir *SD;
+
+ /* Notify the fs that we're not referencing this object any more */
+ if (e->swap_filen > -1) {
+ SD = INDEXSD(e->swap_dirn);
+ if (SD->unrefobj != NULL)
+ SD->unrefobj(SD, e);
+ }
+ /* Notify the memory cache that we're not referencing this object any more */
+ if(e->mem_obj) {
+ if (mem_policy->Dereferenced)
+ mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+
+void
+storeLockObject(StoreEntry * e)
+{
e->lock_count++;
debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
storeKeyText(e->key), (int) e->lock_count);
e->lastref = squid_curtime;
- /* Notify the fs that we're referencing this object again */
- if (SD != NULL && SD->refobj != NULL)
- SD->refobj(SD, e);
+ storeEntryReferenced(e);
}
void
int
storeUnlockObject(StoreEntry * e)
{
- SwapDir *SD;
e->lock_count--;
debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
storeKeyText(e->key), e->lock_count);
if (e->store_status == STORE_PENDING)
EBIT_SET(e->flags, RELEASE_REQUEST);
assert(storePendingNClients(e) == 0);
- /* Notify the fs that we're not referencing this object any more */
- if (e->swap_filen > -1)
- SD = INDEXSD(e->swap_dirn);
- else
- SD = NULL;
- if (SD != NULL && SD->unrefobj != NULL)
- SD->unrefobj(SD, e);
-#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e, SD);
-#else
-#if 0
- /* Note: From 2.4. Not sure how this relates to the unrefobj() call above */
- storeDirLRUDelete(e);
- storeDirLRUAdd(e);
-#endif
-#endif
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
storeRelease(e);
else if (storeKeepInMemory(e)) {
+ storeEntryDereferenced(e);
storeSetMemStatus(e, IN_MEMORY);
requestUnlink(e->mem_obj->request);
e->mem_obj->request = NULL;
} else {
storePurgeMem(e);
+ storeEntryDereferenced(e);
if (EBIT_TEST(e->flags, KEY_PRIVATE))
debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__);
}
* If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
* be set, and storeSetPublicKey() should not be called.
*/
-#if HEAP_REPLACEMENT
+#if MORE_DEBUG_OUTPUT
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n",
e->key, mem->url);
int released = 0;
static time_t last_check = 0;
int pages_needed;
- int locked = 0;
-#if !HEAP_REPLACEMENT
- dlink_node *head;
- dlink_node *m;
- dlink_node *prev = NULL;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#endif
+ RemovalPurgeWalker *walker;
if (squid_curtime == last_check)
return;
last_check = squid_curtime;
if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
return;
debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed);
-#if HEAP_REPLACEMENT
- while (heap_nodes(inmem_heap) > 0) {
- age = heap_peepminkey(inmem_heap);
- e = heap_extractmin(inmem_heap);
- e->mem_obj->node = NULL; /* no longer in the heap */
- if (storeEntryLocked(e)) {
- locked++;
- debug(20, 5) ("storeGetMemSpace: locked key %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
- released++;
- debug(20, 3) ("Released memory object with key %f size %d refs %d url %s\n",
- age, e->swap_file_sz, e->refcount, e->mem_obj->url);
- min_age = age;
+ /* XXX what to set as max_scan here? */
+ walker = mem_policy->PurgeInit(mem_policy, 100000);
+ while((e = walker->Next(walker))) {
storePurgeMem(e);
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
- break;
- }
- /*
- * Increase the heap age factor.
- */
- if (min_age > 0)
- inmem_heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->mem_obj->node = heap_insert(inmem_heap, e);
-#else
- head = inmem_list.head;
- for (m = inmem_list.tail; m; m = prev) {
- if (m == head)
- break;
- prev = m->prev;
- e = m->data;
- if (storeEntryLocked(e)) {
- locked++;
- dlinkDelete(m, &inmem_list);
- dlinkAdd(e, m, &inmem_list);
- continue;
- }
released++;
- storePurgeMem(e);
if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
break;
}
-#endif
- debug(20, 3) ("storeGetMemSpace: released %d/%d locked %d\n",
- released, hot_obj_count, locked);
+ walker->Done(walker);
debug(20, 3) ("storeGetMemSpace stats:\n");
debug(20, 3) (" %6d HOT objects\n", hot_obj_count);
debug(20, 3) (" %6d were released\n", released);
{
int i;
SwapDir *SD;
+ static time_t last_warn_time = 0;
/* walk each fs */
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
/* call the maintain function .. */
- SD = INDEXSD(i);
- if (SD->maintainfs != NULL)
+ SD = INDEXSD(i);
+ /* XXX FixMe: This should be done "in parallell" on the different
+ * cache_dirs, not one at a time.
+ */
+ if (SD->maintainfs != NULL)
SD->maintainfs(SD);
}
-
+ if (store_swap_size > Config.Swap.maxSize) {
+ if (squid_curtime - last_warn_time > 10) {
+ debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
+ store_swap_size, Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
+ }
+ }
/* Reregister a maintain event .. */
eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
}
storeInitHashValues();
store_table = hash_create(storeKeyHashCmp,
store_hash_buckets, storeKeyHashHash);
+ mem_policy = createRemovalPolicy(Config.memPolicy);
storeDigestInit();
storeLogOpen();
-#if HEAP_REPLACEMENT
- inmem_heap = new_heap(1000, HeapKeyGen_StoreEntry_GDSF);
-#endif
stackInit(&LateReleaseStack);
eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
storeDirInit();
assert(mem != NULL);
if (new_status == IN_MEMORY) {
assert(mem->inmem_lo == 0);
-#if HEAP_REPLACEMENT
- if (mem->node == NULL) {
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- debug(20, 4) ("storeSetMemStatus: not inserting special %s\n",
- mem->url);
- } else {
- mem->node = heap_insert(inmem_heap, e);
- debug(20, 4) ("storeSetMemStatus: inserted mem node %p\n",
- mem->node);
- }
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n",
+ mem->url);
+ } else {
+ mem_policy->Add(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n",
+ mem->url);
}
-#else
- dlinkAdd(e, &mem->lru, &inmem_list);
-#endif
hot_obj_count++;
} else {
-#if HEAP_REPLACEMENT
- /*
- * It's being removed from the memory heap; is it already gone?
- */
- if (mem->node) {
- heap_delete(inmem_heap, mem->node);
- debug(20, 4) ("storeSetMemStatus: deleted mem node %p\n",
- mem->node);
- mem->node = NULL;
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: special entry %s\n",
+ mem->url);
+ } else {
+ mem_policy->Remove(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: removed mem node %s\n",
+ mem->url);
}
-#else
- dlinkDelete(&mem->lru, &inmem_list);
-#endif
hot_obj_count--;
}
e->mem_status = new_status;
e->expires = e->lastmod = e->timestamp = -1;
}
-#if HEAP_REPLACEMENT
-/*
- * This routine only handles memory updates these days
- */
-void
-storeHeapPositionUpdate(StoreEntry * e, SwapDir * SD)
-{
- if (e->mem_obj && e->mem_obj->node)
- heap_update(inmem_heap, e->mem_obj->node, e);
-}
-#endif
-
/*
* storeFsInit
*
/*
- * $Id: store_digest.cc,v 1.41 2000/06/06 19:34:31 hno Exp $
+ * $Id: store_digest.cc,v 1.42 2000/06/08 18:05:36 hno Exp $
*
* DEBUG: section 71 Store Digest Manager
* AUTHOR: Alex Rousskov
* idea: skip objects that are going to be purged before the next
* update.
*/
-#if 0 /* This code isn't applicable anymore, we can't fix it atm either :( */
-#if !HEAP_REPLACEMENT
+#if OLD_UNUSED_CODE /* This code isn't applicable anymore, we can't fix it atm either :( */
if ((squid_curtime + Config.digest.rebuild_period) - e->lastref > storeExpiredReferenceAge())
return 0;
-#endif
#endif
return 1;
}
/*
- * $Id: store_dir.cc,v 1.110 2000/05/28 22:15:42 wessels Exp $
+ * $Id: store_dir.cc,v 1.111 2000/06/08 18:05:36 hno Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
#include "squid.h"
static int storeDirValidSwapDirSize(int, ssize_t);
-static void storeDirLRUWalkInitHead(SwapDir * sd);
-static void *storeDirLRUWalkNext(SwapDir * sd);
void
storeDirInit(void)
int
storeDirWriteCleanLogs(int reopen)
{
- StoreEntry *e = NULL;
+ const StoreEntry *e = NULL;
int n = 0;
struct timeval start;
double dt;
SwapDir *sd;
+ RemovalPolicyWalker **walkers;
int dirn;
-#if HEAP_REPLACEMENT
- int node;
-#endif
+ int notdone = 1;
if (store_dirs_rebuilding) {
debug(20, 1) ("Not currently OK to rewrite swap log.\n");
debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n");
debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n");
getCurrentTime();
start = current_time;
+ walkers = xcalloc(Config.cacheSwap.n_configured, sizeof *walkers);
for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
sd = &Config.cacheSwap.swapDirs[dirn];
- if (sd->log.clean.open(sd) < 0) {
- debug(20, 1) ("log.clean.open() failed for dir #%d\n", sd->index);
+ if (sd->log.clean.start(sd) < 0) {
+ debug(20, 1) ("log.clean.start() failed for dir #%d\n", sd->index);
continue;
}
- if (NULL == sd->log.clean.write)
- continue;
-#if HEAP_REPLACEMENT
- if (NULL == sd->repl.heap.heap)
- continue;
-#endif
-#if HEAP_REPLACEMENT
- for (node = 0; node < heap_nodes(sd->repl.heap.heap); node++)
-#else
- storeDirLRUWalkInitHead(sd);
- while ((e = storeDirLRUWalkNext(sd)) != NULL)
-#endif
- {
-#if HEAP_REPLACEMENT
- e = (StoreEntry *) heap_peep(sd->repl.heap.heap, node);
-#endif
+ }
+ while(notdone) {
+ notdone = 0;
+ for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
+ sd = &Config.cacheSwap.swapDirs[dirn];
+ if (NULL == sd->log.clean.write)
+ continue;
+ e = sd->log.clean.nextentry(sd);
+ if (!e)
+ continue;
+ notdone = 1;
if (e->swap_filen < 0)
continue;
if (e->swap_status != SWAPOUT_DONE)
continue;
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
continue;
- sd->log.clean.write(e, sd);
+ sd->log.clean.write(sd, e);
if ((++n & 0xFFFF) == 0) {
getCurrentTime();
debug(20, 1) (" %7d entries written so far.\n", n);
}
}
- /* Flush */
- sd->log.clean.write(NULL, sd);
+ }
+ /* Flush */
+ for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
+ sd = &Config.cacheSwap.swapDirs[dirn];
+ sd->log.clean.done(sd);
}
if (reopen)
storeDirOpenSwapLogs();
void
storeDirCallback(void)
{
- int i;
+ int i, j;
SwapDir *SD;
-
- for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- SD = &Config.cacheSwap.swapDirs[i];
- if (SD->callback != NULL)
- SD->callback(SD);
- }
-}
-
-#if 0 /* from Squid-2.4.DEVEL3 */
-void
-storeDirLRUDelete(StoreEntry * e)
-{
- SwapDir *sd;
- if (e->swap_filen < 0)
- return;
- sd = &Config.cacheSwap.swapDirs[e->swap_dirn];
- dlinkDelete(&e->lru, &sd->repl.lru.list);
-}
-
-void
-storeDirLRUAdd(StoreEntry * e)
-{
- SwapDir *sd;
- if (e->swap_file_number < 0)
- return;
- sd = &Config.cacheSwap.swapDirs[e->swap_dirn];
- dlinkAdd(e, &e->lru, &sd->repl.lru.list);
-}
-#endif /* from Squid-2.4.DEVEL3 */
-
-static void
-storeDirLRUWalkInitHead(SwapDir * sd)
-{
- sd->repl.lru.walker = sd->repl.lru.list.head;
+ static int ndir = 0;
+ do {
+ j = 0;
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ if (ndir >= Config.cacheSwap.n_configured)
+ ndir = ndir % Config.cacheSwap.n_configured;
+ SD = &Config.cacheSwap.swapDirs[ndir++];
+ if (NULL == SD->callback)
+ continue;
+ j += SD->callback(SD);
+ }
+ } while (j > 0);
+ ndir++;
}
-static void *
-storeDirLRUWalkNext(SwapDir * sd)
-{
- void *p;
- if (NULL == sd->repl.lru.walker)
- return NULL;
- p = sd->repl.lru.walker->data;
- sd->repl.lru.walker = sd->repl.lru.walker->next;
- return p;
-}
/*
- * $Id: store_log.cc,v 1.15 2000/05/29 23:30:46 hno Exp $
+ * $Id: store_log.cc,v 1.16 2000/06/08 18:05:36 hno Exp $
*
* DEBUG: section 20 Storage Manager Logging Functions
* AUTHOR: Duane Wessels
* Because if we print it before the swap file number, it'll break
* the existing log format.
*/
- logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %4d %9d %9d %9d %s %d/%d %s %s\n",
+ logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d %s %d/%d %s %s\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
e->swap_dirn,
e->swap_filen,
+ storeKeyText(e->key),
reply->sline.status,
(int) reply->date,
(int) reply->last_modified,
mem->log_url);
} else {
/* no mem object. Most RELEASE cases */
- logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X ? ? ? ? ?/? ?/? ? %s\n",
+ logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s ? ? ? ? ?/? ?/? ? ?\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
/*
- * $Id: structs.h,v 1.338 2000/05/31 07:01:42 hno Exp $
+ * $Id: structs.h,v 1.339 2000/06/08 18:05:36 hno Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
#endif
+struct _RemovalPolicySettings {
+ char *type;
+ wordlist *args;
+};
+
struct _SquidConfig {
struct {
size_t maxSize;
int pct;
size_t max;
} quickAbort;
-#if HEAP_REPLACEMENT
- char *replPolicy;
-#else
- /*
- * Note: the non-LRU policies do not use referenceAge, but we cannot
- * remove it until we find out how to implement #else for cf_parser.c
- */
-#endif
+ RemovalPolicySettings *replPolicy;
+ RemovalPolicySettings *memPolicy;
time_t referenceAge;
time_t negativeTtl;
time_t negativeDnsTtl;
};
+/* Removal policies */
+
+struct _RemovalPolicyNode {
+ void *data;
+};
+
+struct _RemovalPolicy {
+ char *_type;
+ void *_data;
+ void (*Free)(RemovalPolicy *policy);
+ void (*Add)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node);
+ void (*Remove)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node);
+ void (*Referenced)(RemovalPolicy *policy, const StoreEntry *entry, RemovalPolicyNode *node);
+ void (*Dereferenced)(RemovalPolicy *policy, const StoreEntry *entry, RemovalPolicyNode *node);
+ RemovalPolicyWalker *(*WalkInit)(RemovalPolicy *policy);
+ RemovalPurgeWalker *(*PurgeInit)(RemovalPolicy *policy, int max_scan);
+};
+
+struct _RemovalPolicyWalker {
+ RemovalPolicy *_policy;
+ void *_data;
+ const StoreEntry *(*Next)(RemovalPolicyWalker *walker);
+ void (*Done)(RemovalPolicyWalker *walker);
+};
+
+struct _RemovalPurgeWalker {
+ RemovalPolicy *_policy;
+ void *_data;
+ int scanned, max_scan, locked;
+ StoreEntry *(*Next)(RemovalPurgeWalker *walker);
+ void (*Done)(RemovalPurgeWalker *walker);
+};
+
/* This structure can be freed while object is purged out from memory */
struct _MemObject {
method_t method;
void *data;
} abort;
char *log_url;
-#if HEAP_REPLACEMENT
- /*
- * A MemObject knows where it is in the in-memory heap.
- */
- heap_node *node;
-#else
- dlink_node lru;
-#endif
+ RemovalPolicyNode repl;
int id;
ssize_t object_sz;
size_t swap_hdr_sz;
u_short flags;
sdirno swap_dirn;
sfileno swap_filen;
- union {
-#ifdef HEAP_REPLACEMENT
- heap_node *node;
-#endif
- dlink_node lru;
- } repl;
+ RemovalPolicyNode repl;
u_short lock_count; /* Assume < 65536! */
mem_status_t mem_status:3;
ping_status_t ping_status:3;
int index; /* This entry's index into the swapDirs array */
int suggest;
ssize_t max_objsize;
- union {
-#ifdef HEAP_REPLACEMENT
- struct {
- heap *heap;
- } heap;
-#endif
- struct {
- dlink_list list;
- dlink_node *walker;
- } lru;
- } repl;
+ RemovalPolicy *repl;
int removals;
int scanned;
struct {
STLOGCLOSE *close;
STLOGWRITE *write;
struct {
- STLOGCLEANOPEN *open;
+ STLOGCLEANSTART *start;
+ STLOGCLEANNEXTENTRY *nextentry;
STLOGCLEANWRITE *write;
+ STLOGCLEANDONE *done;
void *state;
} clean;
} log;
size_t bufsz;
off_t offset;
};
+
/*
- * $Id: typedefs.h,v 1.103 2000/05/12 00:29:09 wessels Exp $
+ * $Id: typedefs.h,v 1.104 2000/06/08 18:05:37 hno Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
typedef struct _storefs_entry storefs_entry_t;
typedef struct _diskd_queue diskd_queue;
typedef struct _Logfile Logfile;
+typedef struct _RemovalPolicy RemovalPolicy;
+typedef struct _RemovalPolicyWalker RemovalPolicyWalker;
+typedef struct _RemovalPurgeWalker RemovalPurgeWalker;
+typedef struct _RemovalPolicyNode RemovalPolicyNode;
+typedef struct _RemovalPolicySettings RemovalPolicySettings;
#if SQUID_SNMP
typedef variable_list *(oid_ParseFn) (variable_list *, snint *);
typedef void STUNREFOBJ(SwapDir *, StoreEntry *);
typedef void STSETUP(storefs_entry_t *);
typedef void STDONE(void);
-typedef void STCALLBACK(SwapDir *);
+typedef int STCALLBACK(SwapDir *);
typedef void STSYNC(SwapDir *);
typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *);
typedef void STLOGOPEN(SwapDir *);
typedef void STLOGCLOSE(SwapDir *);
typedef void STLOGWRITE(const SwapDir *, const StoreEntry *, int);
-typedef int STLOGCLEANOPEN(SwapDir *);
-typedef void STLOGCLEANWRITE(const StoreEntry *, SwapDir *);
+typedef int STLOGCLEANSTART(SwapDir *);
+typedef const StoreEntry *STLOGCLEANNEXTENTRY(SwapDir *);
+typedef void STLOGCLEANWRITE(SwapDir *, const StoreEntry *);
+typedef void STLOGCLEANDONE(SwapDir *);
/* Store dir configuration routines */
/* SwapDir *sd, char *path ( + char *opt later when the strtok mess is gone) */
typedef struct _htcpReplyData htcpReplyData;
#endif
+typedef RemovalPolicy *REMOVALPOLICYCREATE(wordlist *args);
#endif /* _TYPEDEFS_H_ */
projects / thirdparty / squid.git / commitdiff
