From: robertc <> Date: Mon, 3 Jan 2005 23:08:24 +0000 (+0000) Subject: Merge in current state of store refactoring work. The search method has been X-Git-Tag: SQUID_3_0_PRE4~921 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c8f4eac4b12f831b82d87754a5a134e0fbd4108e;p=thirdparty%2Fsquid.git Merge in current state of store refactoring work. The search method has been fully implemented for all current store types. Patches applied: * robertc@squid-cache.org--squid/squid--store-interface--3.0--base-0 tag of robertc@squid-cache.org--squid/squid--HEAD--3.0--patch-516 * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-1 merge in the diskio logic cleanup we depend on * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-2 setup a unit test environment for store.cc * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-3 merge from HEAD * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-4 start formal store interface refactoring, moving stat into the Store hierarchy * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-5 make SwapDirs reference counted, test StoreControllers new methods * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-6 move storeDirInit to be just the virtual init method on Store * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-7 nuke Config.Swap.maxSize for calls to Store::Root().maxSize() * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-8 migrate Reference and dereference to StoreController * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-9 make store maintenance a top level Store api call * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-10 move storeDirUpdateSize to Store interface * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-11 move store creation into the Store API * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-12 move storeDirSync into the Store API * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-13 more INDEXSD nuking, and push the swapdir aspects of store statting into SwapDir * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-14 more INDEXSD nuking. move SwapDir.low_size to a minSize() query * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-15 another INDEXSD removal, storeDirCallback folded into the Store API * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-16 correct logic error in StoreController::Callback from previous changes * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-17 storeUnlink -> StoreEntry::unlink * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-18 snapshot, beginning the introduction of a separate HashIndex store layer * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-19 clean up store entry construction a little * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-20 setup Ufs testing harness * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-22 make RemovalPolicySettingsa class * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-23 make testUfs use a real UFSSwapDir * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-24 testUfs should clean up after it succeeds * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-25 merge from HEAD * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-26 wave enough dead chickens that we can set a store object to public * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-27 get testUfs to put an object in the store successfully * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-28 get testUfs to save clean log state * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-29 let rebuild work fully on testUfs, so we can start adapting RebuildState to being a StoreSearch * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-30 implement basic search for UFSSwapDirs * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-31 check UFS search iteration stops * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-32 convert RebuildState to CDBATA_CLASS2 * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-33 switch from log based searching to replacement policy based searching for UFSSwapDirs * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-34 rename variable s to swapData in UFS restore from directory routine * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-35 store rebuilds need to check for duplicates across the entire store. * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-36 start testing COSS - store creation for starters * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-37 make StoreSearch a header of its own, and implement search(NULL,NULL) for COSS * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-38 start testing the Null store type * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-39 implement null store search(NULL, NULL) * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-40 remove a couple of unneded SwapDir.h includes * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-41 move search one level up in the Store hierarchy * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-42 test the rudiments of StoreHashIndex * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-43 move hash initialisation into StoreHashIndex::init * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-44 oops, forgot new files * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-45 implement search for StoreHashIndex * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-46 void * apis suck badly. fix a casting problem with destroyStoreEntry * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-47 implement search for StoreController * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-48 convert store object statting to use the StoreSearch api * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-49 convert cache digests to the store search api * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-50 convert store cleanup callback to use the store api * robertc@squid-cache.org--squid/squid--store-interface--3.0--patch-51 merge from HEAD --- diff --git a/ChangeLog b/ChangeLog index e6b631a4f5..97e6fa888d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes to squid-2.6 (): +Changes to squid-3.0 (): - CARP now plays well with the other peering algorithms, and support for CARP peerings is compiled by default. Can be @@ -14,7 +14,9 @@ Changes to squid-2.6 (): occurred error by default, so that the email will contain this data in its body. This feature can be disabled via the email_err_data directive. (Clemens Löser) - + - COSS now uses a file called stripe and the path in squid.conf is the + directory this is placed in. Additionally squid -z will create the + COSS swapfile. Changes to squid-2.5 (): diff --git a/include/Array.h b/include/Array.h index 9b48e581d9..0c8a944034 100644 --- a/include/Array.h +++ b/include/Array.h @@ -1,5 +1,5 @@ /* - * $Id: Array.h,v 1.19 2004/12/20 16:30:29 robertc Exp $ + * $Id: Array.h,v 1.20 2005/01/03 16:08:24 robertc Exp $ * * AUTHOR: Alex Rousskov * @@ -101,6 +101,7 @@ public: const_iterator begin () const; iterator end(); const_iterator end () const; + E& operator [] (unsigned i); /* Do not change these, until the entry C struct is removed */ size_t capacity; @@ -282,6 +283,13 @@ Vector::end() const return const_iterator(size(), *this); } +template +E & +Vector::operator [] (unsigned i) +{ + assert (size() > i); + return items[i]; +} template VectorIteratorBase::VectorIteratorBase() : pos(0), theVector(NULL) diff --git a/src/DiskIO/DiskDaemon/DiskdIOStrategy.cc b/src/DiskIO/DiskDaemon/DiskdIOStrategy.cc index 49de2399ed..799f394907 100644 --- a/src/DiskIO/DiskDaemon/DiskdIOStrategy.cc +++ b/src/DiskIO/DiskDaemon/DiskdIOStrategy.cc @@ -1,6 +1,6 @@ /* - * $Id: DiskdIOStrategy.cc,v 1.1 2004/12/20 16:30:38 robertc Exp $ + * $Id: DiskdIOStrategy.cc,v 1.2 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 79 Squid-side DISKD I/O functions. * AUTHOR: Duane Wessels @@ -397,8 +397,8 @@ DiskdIOStrategy::send(int mtype, int id, DiskdFile *theFile, int size, int offse * up to 2*magic1) and we can run out of shared memory buffers. */ /* - * Note that we call storeDirCallback (for all SDs), rather - * than storeDiskdDirCallback for just this SD, so that while + * Note that we call Store::Root().callbackk (for all SDs), rather + * than callback for just this SD, so that while * we're "blocking" on this SD we can also handle callbacks * from other SDs that might be ready. */ @@ -407,7 +407,7 @@ DiskdIOStrategy::send(int mtype, int id, DiskdFile *theFile, int size, int offse struct timeval delay = {0, 1}; select(0, NULL, NULL, NULL, &delay); - storeDirCallback(); + Store::Root().callback(); if (delay.tv_usec < 1000000) delay.tv_usec <<= 1; @@ -456,8 +456,8 @@ DiskdIOStrategy::send(int mtype, int id, StoreIOState::Pointer sio, int size, in * up to 2*magic1) and we can run out of shared memory buffers. */ /* - * Note that we call storeDirCallback (for all SDs), rather - * than storeDiskdDirCallback for just this SD, so that while + * Note that we call Store::Root().callbackk (for all SDs), rather + * than callback for just this SD, so that while * we're "blocking" on this SD we can also handle callbacks * from other SDs that might be ready. */ @@ -466,7 +466,7 @@ DiskdIOStrategy::send(int mtype, int id, StoreIOState::Pointer sio, int size, in struct timeval delay = {0, 1}; select(0, NULL, NULL, NULL, &delay); - storeDirCallback(); + Store::Root().callback(); if (delay.tv_usec < 1000000) delay.tv_usec <<= 1; diff --git a/src/DiskIO/DiskThreads/DiskThreadsDiskFile.cc b/src/DiskIO/DiskThreads/DiskThreadsDiskFile.cc index bd4520d31a..1694aae6c5 100644 --- a/src/DiskIO/DiskThreads/DiskThreadsDiskFile.cc +++ b/src/DiskIO/DiskThreads/DiskThreadsDiskFile.cc @@ -1,6 +1,6 @@ /* - * $Id: DiskThreadsDiskFile.cc,v 1.2 2004/12/21 17:28:29 robertc Exp $ + * $Id: DiskThreadsDiskFile.cc,v 1.3 2005/01/03 16:08:27 robertc Exp $ * * DEBUG: section 79 Disk IO Routines * AUTHOR: Robert Collins @@ -38,7 +38,6 @@ #include "squid.h" #include "DiskThreadsDiskFile.h" #include "Store.h" -#include "SwapDir.h" #include "Generic.h" #include "DiskIO/IORequestor.h" #include "DiskIO/ReadRequest.h" diff --git a/src/Makefile.am b/src/Makefile.am index 77e010ffdf..1bb00379b7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.am,v 1.102 2004/12/21 18:20:19 robertc Exp $ +# $Id: Makefile.am,v 1.103 2005/01/03 16:08:25 robertc Exp $ # # Uncomment and customize the following to suit your needs: # @@ -217,6 +217,18 @@ all_DISKIOMODULES = \ DiskIO/DiskDaemon/DiskDaemonDiskIOModule.cc \ DiskIO/DiskThreads/DiskThreadsDiskIOModule.cc +DISKIO_SOURCE = \ + DiskIO/DiskIOModule.cc \ + DiskIO/ReadRequest.cc \ + DiskIO/ReadRequest.h \ + DiskIO/WriteRequest.cc \ + DiskIO/WriteRequest.h \ + DiskIO/DiskFile.h \ + DiskIO/DiskIOStrategy.h \ + DiskIO/IORequestor.h \ + DiskIO/DiskIOModule.h \ + DiskIO/ReadRequest.h + all_AUTHMODULES = \ auth/basic/basicScheme.cc \ auth/basic/basicScheme.h \ @@ -364,11 +376,7 @@ squid_SOURCES = \ defines.h \ $(DELAY_POOL_SOURCE) \ disk.cc \ - DiskIO/DiskIOModule.cc \ - DiskIO/ReadRequest.cc \ - DiskIO/ReadRequest.h \ - DiskIO/WriteRequest.cc \ - DiskIO/WriteRequest.h \ + $(DISKIO_SOURCE) \ $(DNSSOURCE) \ $(EPOLL_SOURCE) \ enums.h \ @@ -427,6 +435,7 @@ squid_SOURCES = \ IPInterception.cc \ IPInterception.h \ $(LEAKFINDERSOURCE) \ + list.cc \ logfile.cc \ main.cc \ mem.cc \ @@ -441,6 +450,8 @@ squid_SOURCES = \ neighbors.cc \ net_db.cc \ Packer.cc \ + Parsing.cc \ + Parsing.h \ $(XPROF_STATS_SOURCE) \ pconn.cc \ peer_digest.cc \ @@ -466,6 +477,7 @@ squid_SOURCES = \ Store.h \ StoreFileSystem.cc \ StoreFileSystem.h \ + StoreHashIndex.h \ store_io.cc \ StoreIOBuffer.h \ StoreIOState.cc \ @@ -492,6 +504,7 @@ squid_SOURCES = \ StoreMetaURL.h \ StoreMetaVary.cc \ StoreMetaVary.h \ + StoreSearch.h \ StoreSwapLogData.cc \ StoreSwapLogData.h \ structs.h \ @@ -683,6 +696,7 @@ ufsdump_SOURCES = debug.cc \ neighbors.cc \ net_db.cc \ Packer.cc \ + Parsing.cc \ $(XPROF_STATS_SOURCE) \ pconn.cc \ peer_digest.cc \ @@ -766,13 +780,6 @@ data_DATA = \ LDADD = -L../lib -lmiscutil @XTRA_LIBS@ @EPOLL_LIBS@ -DISKIODIST = \ - DiskIO/DiskFile.h \ - DiskIO/DiskIOStrategy.h \ - DiskIO/IORequestor.h \ - DiskIO/DiskIOModule.h \ - DiskIO/ReadRequest.h - EXTRA_DIST = \ cf_gen_defines \ cf.data.pre \ @@ -780,8 +787,7 @@ EXTRA_DIST = \ mk-string-arrays.pl \ repl_modules.sh \ mib.txt \ - mime.conf.default \ - $(DISKIODIST) + mime.conf.default libAIO_a_SOURCES = \ DiskIO/AIO/async_io.h \ @@ -939,7 +945,7 @@ DISTCLEANFILES = cf_gen_defines.h cf.data cf_parser.h squid.conf.default \ TESTSOURCES=../test-suite/test_tools.cc -check_PROGRAMS+=tests/testAuth +check_PROGRAMS+=tests/testAuth tests/testStore tests/testUfs tests/testCoss tests/testNull tests_testAuth_SOURCES= tests/testAuth.cc tests/testMain.cc tests/testAuth.h $(TESTSOURCES) \ AuthScheme.cc globals.cc authenticate.cc AuthUser.cc AuthUserRequest.cc AuthConfig.cc \ tests/stub_acl.cc tests/stub_cache_cf.cc \ @@ -964,3 +970,134 @@ tests_testAuth_DEPENDENCIES = $(top_builddir)/lib/libmiscutil.a \ @AUTH_LINKOBJS@ \ @AUTH_OBJS@ \ @SQUID_CPPUNIT_LA@ + + +STORE_TEST_SOURCES=\ + $(TESTSOURCES) \ + CacheDigest.cc \ + store_dir.cc \ + store.cc \ + store_key_md5.cc \ + globals.cc \ + string_arrays.c \ + Parsing.cc \ + ConfigOption.cc \ + SwapDir.cc \ + AuthScheme.cc authenticate.cc AuthUser.cc AuthUserRequest.cc AuthConfig.cc \ + tests/stub_acl.cc tests/stub_cache_cf.cc \ + tests/stub_helper.cc cbdata.cc String.cc tests/stub_cache_manager.cc \ + tests/stub_comm.cc \ + tests/stub_http.cc \ + mem_node.cc \ + stmem.cc \ + tests/stub_store_client.cc \ + tests/stub_tools.cc \ + HttpHeaderTools.cc HttpHeader.cc acl.cc event.cc mem.cc \ + MemBuf.cc HttpHdrContRange.cc Packer.cc ACLChecklist.cc HttpHdrCc.cc HttpHdrSc.cc \ + HttpHdrScTarget.cc url.cc ACLProxyAuth.cc ACLRegexData.cc ACLUserData.cc \ + StatHist.cc HttpHdrRange.cc ETag.cc tests/stub_errorpage.cc \ + tests/stub_HttpRequest.cc + +## why so many sources? well httpHeaderTools requites ACLChecklist & friends. +## first line - what we are testing. +tests_testStore_SOURCES= tests/testStore.cc \ + tests/testStoreController.cc \ + tests/testStoreHashIndex.cc \ + tests/testMain.cc \ + tests/testStore.h \ + tests/testStoreController.h \ + tests/testStoreHashIndex.h \ + tests/TestSwapDir.cc \ + tests/TestSwapDir.h \ + tests/stub_store_swapout.cc \ + tests/stub_fd.cc \ + tests/stub_HttpReply.cc \ + tests/stub_MemObject.cc \ + $(STORE_TEST_SOURCES) + +tests_testStore_LDADD= \ + -L../lib -lmiscutil \ + @SQUID_CPPUNIT_LA@ +tests_testStore_LDFLAGS = $(LIBADD_DL) +tests_testStore_DEPENDENCIES = $(top_builddir)/lib/libmiscutil.a \ + @SQUID_CPPUNIT_LA@ + +SWAP_TEST_SOURCES=\ + tests/stub_store_rebuild.cc \ + tests/stub_internal.cc \ + fd.cc \ + disk.cc \ + filemap.cc \ + HttpBody.cc \ + HttpMsg.cc \ + HttpReply.cc \ + HttpStatusLine.cc \ + int.cc \ + list.cc \ + MemObject.cc \ + mime.cc \ + StoreSwapLogData.cc \ + StoreIOState.cc \ + StoreMeta.cc \ + StoreMetaMD5.cc \ + StoreMetaSTD.cc \ + StoreMetaUnpacker.cc \ + StoreMetaURL.cc \ + StoreMetaVary.cc \ + StoreFileSystem.cc \ + store_io.cc \ + store_swapout.cc \ + store_swapmeta.cc \ + unlinkd.cc \ + $(STORE_TEST_SOURCES) \ + $(DISKIO_SOURCE) + +SWAP_TEST_LDADD = \ + DiskIO/Blocking/BlockingDiskIOModule.o \ + repl/lru/store_repl_lru.o \ + @STORE_LINKOBJS@ \ + @STORE_OBJS@ \ + @REPL_OBJS@ \ + @DISK_LIBS@ \ + -L../lib -lmiscutil \ + @SQUID_CPPUNIT_LA@ +SWAP_TEST_DS =\ + $(top_builddir)/lib/libmiscutil.a \ + DiskIO/Blocking/BlockingDiskIOModule.o \ + repl/lru/store_repl_lru.o \ + repl_modules.o \ + @DISK_LIBS@ \ + @STORE_LINKOBJS@ \ + @STORE_OBJS@ \ + @REPL_OBJS@ \ + @SQUID_CPPUNIT_LA@ + +tests_testUfs_SOURCES= tests/testUfs.cc \ + tests/testMain.cc \ + tests/testUfs.h \ + $(SWAP_TEST_SOURCES) +tests_testUfs_LDADD= \ + $(SWAP_TEST_LDADD) +tests_testUfs_LDFLAGS = $(LIBADD_DL) +tests_testUfs_DEPENDENCIES = \ + $(SWAP_TEST_DS) + +tests_testCoss_SOURCES= tests/testCoss.cc \ + tests/testMain.cc \ + tests/testCoss.h \ + $(SWAP_TEST_SOURCES) +tests_testCoss_LDADD= \ + $(SWAP_TEST_LDADD) +tests_testCoss_LDFLAGS = $(LIBADD_DL) +tests_testCoss_DEPENDENCIES = \ + $(SWAP_TEST_DS) + +tests_testNull_SOURCES= tests/testNull.cc \ + tests/testMain.cc \ + tests/testNull.h \ + $(SWAP_TEST_SOURCES) +tests_testNull_LDADD= \ + $(SWAP_TEST_LDADD) +tests_testNull_LDFLAGS = $(LIBADD_DL) +tests_testNull_DEPENDENCIES = \ + $(SWAP_TEST_DS) diff --git a/src/MemObject.cc b/src/MemObject.cc index 116f7183d0..6b19f30efd 100644 --- a/src/MemObject.cc +++ b/src/MemObject.cc @@ -1,6 +1,6 @@ /* - * $Id: MemObject.cc,v 1.14 2004/12/27 11:04:36 serassio Exp $ + * $Id: MemObject.cc,v 1.15 2005/01/03 16:08:25 robertc Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Robert Collins @@ -110,7 +110,8 @@ MemObject::~MemObject() #endif - httpReplyDestroy((HttpReply *)_reply); + if (_reply) + httpReplyDestroy((HttpReply *)_reply); requestUnlink(request); diff --git a/src/Parsing.cc b/src/Parsing.cc new file mode 100644 index 0000000000..6e7e5a7b68 --- /dev/null +++ b/src/Parsing.cc @@ -0,0 +1,74 @@ + +/* + * $Id: Parsing.cc,v 1.1 2005/01/03 16:08:25 robertc Exp $ + * + * DEBUG: section 3 Configuration File Parsing + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "Parsing.h" + +/* + * These functions is the same as atoi/l/f, except that they check for errors + */ + +long +xatol(const char *token) +{ + char *end; + long ret = strtol(token, &end, 10); + + if (ret == 0 && end == token) + self_destruct(); + + return ret; +} + +int +xatoi(const char *token) +{ + return xatol(token); +} + +int +GetInteger(void) +{ + char *token = strtok(NULL, w_space); + int i; + + if (token == NULL) + self_destruct(); + + if (sscanf(token, "%d", &i) != 1) + self_destruct(); + + return i; +} + diff --git a/src/Parsing.h b/src/Parsing.h new file mode 100644 index 0000000000..e4f5d599e7 --- /dev/null +++ b/src/Parsing.h @@ -0,0 +1,45 @@ + +/* + * $Id: Parsing.h,v 1.1 2005/01/03 16:08:25 robertc Exp $ + * + * DEBUG: section 3 Configuration File Parsing + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_PARSING_H +#define SQUID_PARSING_H + +#include "squid.h" + +extern long xatol(const char *token); +extern int xatoi(const char *token); +extern int GetInteger(void); + +#endif /* SQUID_PARSING_H */ diff --git a/src/Store.cci b/src/Store.cci index c007f9458b..ac41ab7724 100644 --- a/src/Store.cci +++ b/src/Store.cci @@ -1,6 +1,6 @@ /* - * $Id: Store.cci,v 1.1 2003/01/23 00:37:14 robertc Exp $ + * $Id: Store.cci,v 1.2 2005/01/03 16:08:25 robertc Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -57,5 +57,14 @@ StoreEntry::isEmpty () const HttpReply const * NullStoreEntry::getReply() const { - return NULL; + return NULL; +} + +Store & +Store::Root() +{ + if (CurrentRoot == NULL) + fatal("No Store Root has been set"); + + return *CurrentRoot; } diff --git a/src/Store.h b/src/Store.h index 49fa79f726..ea886e7860 100644 --- a/src/Store.h +++ b/src/Store.h @@ -1,6 +1,6 @@ /* - * $Id: Store.h,v 1.13 2004/08/30 05:12:31 robertc Exp $ + * $Id: Store.h,v 1.14 2005/01/03 16:08:25 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -36,6 +36,7 @@ #include "StoreIOBuffer.h" #include "Range.h" +#include "RefCount.h" #include "CommRead.h" #if ESI @@ -46,6 +47,10 @@ class StoreClient; class MemObject; +class Store; + +class StoreSearch; + class StoreEntry : public hash_link { @@ -54,6 +59,8 @@ public: bool checkDeferRead(int fd) const; virtual const char *getMD5Text() const; + StoreEntry(); + StoreEntry(const char *url, const char *log_url); virtual ~StoreEntry(){} virtual HttpReply const *getReply() const; @@ -65,11 +72,14 @@ public: virtual char const *getSerialisedMetaData(); virtual bool swapoutPossible(); virtual void trimMemory(); + void unlink(); void delayAwareRead(int fd, char *buf, int len, IOCB *handler, void *data); void setNoDelay (bool const); bool modifiedSince(HttpRequest * request) const; + /* what store does this entry belong too ? */ + virtual RefCount store() const; MemObject *mem_obj; RemovalPolicyNode repl; @@ -160,14 +170,86 @@ private: static NullStoreEntry _instance; }; +typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata); + +/* Abstract base class that will replace the whole store and swapdir interface. */ + +class Store : public RefCountable +{ + +public: + /* The root store */ + static Store &Root(); + static void Root(Store *); + static void Root(RefCount); + static void Stats(StoreEntry * output); + static void Maintain(void *unused); + + virtual ~Store() {} + + /* Handle pending callbacks - called by the event loop. */ + virtual int callback() = 0; + /* create the resources needed for this store to operate */ + virtual void create(); + /* notify this store that its disk is full. TODO XXX move into a protected api call + * between store files and their stores, rather than a top level api call + */ + virtual void diskFull(); + /* Retrieve a store entry from the store */ + + virtual StoreEntry * get + (const cache_key *) = 0; + + /* TODO: imeplement the async version */ + virtual void get + (String const key , STOREGETCLIENT callback, void *cbdata) = 0; + + /* prepare the store for use. The store need not be usable immediately, + * it should respond to readable() and writable() with true as soon + * as it can provide those services + */ + virtual void init() = 0; + + /* the maximum size the store will support in normal use. Inaccuracy is permitted, + * but may throw estimates for memory etc out of whack. */ + virtual size_t maxSize() const = 0; + + /* The minimum size the store will shrink to via normal housekeeping */ + virtual size_t minSize() const = 0; + + /* TODO: make these calls asynchronous */ + virtual void stat(StoreEntry &) const = 0; /* output stats to the provided store entry */ + + virtual void sync(); /* Sync the store prior to shutdown */ + + /* remove a Store entry from the store */ + virtual void unlink (StoreEntry &); + + /* search in the store */ + virtual StoreSearch *search(String const url, HttpRequest *) = 0; + + /* pulled up from SwapDir for migration.... probably do not belong here */ + virtual void reference(StoreEntry &) = 0; /* Reference this object */ + + virtual void dereference(StoreEntry &) = 0; /* Unreference this object */ + + virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */ + + /* These should really be private */ + virtual void updateSize(size_t size, int sign) = 0; + +private: + static RefCount CurrentRoot; +}; + +typedef RefCount StorePointer; + SQUIDCEXTERN size_t storeEntryInUse(); SQUIDCEXTERN off_t storeLowestMemReaderOffset(const StoreEntry * entry); SQUIDCEXTERN const char *storeEntryFlags(const StoreEntry *); SQUIDCEXTERN int storeEntryLocked(const StoreEntry *); extern void storeEntryReplaceObject(StoreEntry *, HttpReply *); -SQUIDCEXTERN StoreEntry *new_StoreEntry(int, const char *, const char *); -SQUIDCEXTERN StoreEntry *storeGet(const cache_key *); SQUIDCEXTERN StoreEntry *storeGetPublic(const char *uri, const method_t method); SQUIDCEXTERN StoreEntry *storeGetPublicByRequest(HttpRequest * request); SQUIDCEXTERN StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const method_t method); @@ -180,7 +262,6 @@ SQUIDCEXTERN void storeAppend(StoreEntry *, const char *, int); SQUIDCEXTERN void storeLockObject(StoreEntry *); SQUIDCEXTERN void storeRelease(StoreEntry *); SQUIDCEXTERN int storeUnlockObject(StoreEntry *); -SQUIDCEXTERN EVH storeMaintainSwapSpace; SQUIDCEXTERN void storeExpireNow(StoreEntry *); SQUIDCEXTERN void storeReleaseRequest(StoreEntry *); SQUIDCEXTERN void storeConfigure(void); @@ -216,6 +297,7 @@ SQUIDCEXTERN void storeSwapFileNumberSet(StoreEntry * e, sfileno filn); SQUIDCEXTERN void storeFsInit(void); SQUIDCEXTERN void storeFsDone(void); SQUIDCEXTERN void storeReplAdd(const char *, REMOVALPOLICYCREATE *); +extern FREE destroyStoreEntry; #ifdef _USE_INLINE_ #include "Store.cci" diff --git a/src/StoreHashIndex.h b/src/StoreHashIndex.h new file mode 100644 index 0000000000..a34f0cb17b --- /dev/null +++ b/src/StoreHashIndex.h @@ -0,0 +1,122 @@ + +/* + * $Id: StoreHashIndex.h,v 1.1 2005/01/03 16:08:25 robertc Exp $ + * + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_STOREHASHINDEX_H +#define SQUID_STOREHASHINDEX_H + +#include "Store.h" +#include "StoreSearch.h" + +/* A summary store that indexs all its children + * into a memory hash + */ + +class StoreSearch; + +class StoreHashIndex : public Store +{ + +public: + StoreHashIndex(); + StoreHashIndex(StoreHashIndex const &); /* to cause link failures */ + virtual ~StoreHashIndex(); + virtual int callback(); + virtual void create(); + + virtual StoreEntry * get + (const cache_key *); + + virtual void get + (String const, STOREGETCLIENT, void * cbdata); + + virtual void init(); + + virtual void sync(); + + virtual size_t maxSize() const; + + virtual size_t minSize() const; + + virtual void stat(StoreEntry&) const; + + virtual void reference(StoreEntry&); + + virtual void dereference(StoreEntry&); + + virtual void maintain(); + + virtual void updateSize(unsigned int, int); + + virtual StoreSearch *search(String const url, HttpRequest *); + +private: + /* migration logic */ + StorePointer store(int const x) const; +}; + +class StoreHashIndexEntry : public StoreEntry +{} + +; + +class StoreSearchHashIndex : public StoreSearch +{ + +public: + StoreSearchHashIndex(RefCount sd); + StoreSearchHashIndex(StoreSearchHashIndex const &); + virtual ~StoreSearchHashIndex(); + /* Iterator API - garh, wrong place */ + /* callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + /* return true if a new StoreEntry is immediately available */ + virtual bool next(); + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + +private: + CBDATA_CLASS2(StoreSearchHashIndex); + void copyBucket(); + void (*callback)(void *cbdata); + void *cbdata; + bool _done; + int bucket; + Vector entries; +}; + +#endif /* SQUID_STOREHASHINDEX_H */ diff --git a/src/StoreSearch.h b/src/StoreSearch.h new file mode 100644 index 0000000000..1f3e10f11e --- /dev/null +++ b/src/StoreSearch.h @@ -0,0 +1,70 @@ + +/* + * $Id: StoreSearch.h,v 1.1 2005/01/03 16:08:25 robertc Exp $ + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_STORESEARCH_H +#define SQUID_STORESEARCH_H + +#include "squid.h" +#include "RefCount.h" +#include "Store.h" + +class StoreSearch : public RefCountable +{ + +public: + StoreSearch() {} + + StoreSearch(StoreSearch const &); /* no implementation - trigger link failures */ + virtual ~StoreSearch() {} + + /* not ready yet + void asList(void (*) (List StoreSearchPointer; + +#endif /* SQUID_STORESEARCH_H */ diff --git a/src/String.cc b/src/String.cc index 8f7251e8df..b6551459b2 100644 --- a/src/String.cc +++ b/src/String.cc @@ -1,6 +1,6 @@ /* - * $Id: String.cc,v 1.18 2004/08/30 03:28:57 robertc Exp $ + * $Id: String.cc,v 1.19 2005/01/03 16:08:25 robertc Exp $ * * DEBUG: section 67 String * AUTHOR: Duane Wessels @@ -376,6 +376,12 @@ error: return (char *) word; } +const char * +checkNullString(const char *p) +{ + return p ? p : "(NULL)"; +} + #ifndef _USE_INLINE_ #include "String.cci" #endif diff --git a/src/SwapDir.cc b/src/SwapDir.cc index 0e12241eec..46cd6eb916 100644 --- a/src/SwapDir.cc +++ b/src/SwapDir.cc @@ -1,6 +1,6 @@ /* - * $Id: SwapDir.cc,v 1.6 2004/12/20 16:30:34 robertc Exp $ + * $Id: SwapDir.cc,v 1.7 2005/01/03 16:08:25 robertc Exp $ * * DEBUG: section ?? Swap Dir base object * AUTHOR: Robert Collins @@ -35,7 +35,6 @@ #include "squid.h" #include "SwapDir.h" -#include "Store.h" #include "StoreFileSystem.h" #include "ConfigOption.h" @@ -45,7 +44,7 @@ SwapDir::~SwapDir() } void -SwapDir::newFileSystem(){} +SwapDir::create(){} void SwapDir::dump(StoreEntry &)const{} @@ -59,11 +58,35 @@ SwapDir::doubleCheck(StoreEntry &) void SwapDir::unlink(StoreEntry &){} +void +SwapDir::stat(StoreEntry &output) const +{ + storeAppendPrintf(&output, "Store Directory #%d (%s): %s\n", index, type(), + path); + storeAppendPrintf(&output, "FS Block Size %d Bytes\n", + fs.blksize); + statfs(output); + + if (repl) { + storeAppendPrintf(&output, "Removal policy: %s\n", repl->_type); + + if (repl->Stats) + repl->Stats(repl, &output); + } +} + void SwapDir::statfs(StoreEntry &)const {} void -SwapDir::maintainfs(){} +SwapDir::maintain(){} + +size_t +SwapDir::minSize() const +{ + return (size_t) (((float) maxSize() * + (float) Config.Swap.lowWaterMark) / 100.0); +} void SwapDir::reference(StoreEntry &){} @@ -239,3 +262,22 @@ SwapDir::optionMaxSizeDump(StoreEntry * e) const if (max_objsize != -1) storeAppendPrintf(e, " max-size=%ld", (long int) max_objsize); } + +/* Swapdirs do not have an index of their own - thus they ask their parent.. + * but the parent child relationship isn't implemented yet + */ +StoreEntry * + +SwapDir::get + (const cache_key *key) +{ + return Store::Root().get(key); +} + +void + +SwapDir::get + (String const key, STOREGETCLIENT callback, void *cbdata) +{ + fatal("not implemented"); +} diff --git a/src/SwapDir.h b/src/SwapDir.h index a4d2e98fc3..db0c87ffe5 100644 --- a/src/SwapDir.h +++ b/src/SwapDir.h @@ -1,6 +1,6 @@ /* - * $Id: SwapDir.h,v 1.7 2004/12/20 16:30:34 robertc Exp $ + * $Id: SwapDir.h,v 1.8 2005/01/03 16:08:25 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -34,6 +34,7 @@ #ifndef SQUID_SWAPDIR_H #define SQUID_SWAPDIR_H +#include "Store.h" #include "StoreIOState.h" /* Store dir configuration routines */ @@ -41,19 +42,105 @@ class ConfigOption; -class SwapDir +/* New class that replaces the static SwapDir methods as part of the Store overhaul */ + +class StoreController : public Store { public: - SwapDir(char const *aType) : theType (aType), cur_size (0), low_size(0), max_size(0), max_objsize (-1) + StoreController(); + virtual ~StoreController(); + virtual int callback(); + virtual void create(); + + virtual StoreEntry * get + (const cache_key *); + + virtual void get + (String const, STOREGETCLIENT, void * cbdata); + + virtual void init(); + + virtual void maintain(); /* perform regular maintenance should be private and self registered ... */ + + virtual size_t maxSize() const; + + virtual size_t minSize() const; + + virtual void stat(StoreEntry &) const; + + virtual void sync(); /* Sync the store prior to shutdown */ + + virtual StoreSearch *search(String const url, HttpRequest *); + + virtual void reference(StoreEntry &); /* Reference this object */ + + virtual void dereference(StoreEntry &); /* Unreference this object */ + + virtual void updateSize(size_t size, int sign); + +private: + void createOneStore(Store &aStore); + + StorePointer swapDir; +}; + +/* migrating from the Config based list of swapdirs */ +extern void allocate_new_swapdir(_SquidConfig::_cacheSwap *); +extern void free_cachedir(_SquidConfig::_cacheSwap * swap); +SQUIDCEXTERN OBJH storeDirStats; +SQUIDCEXTERN char *storeDirSwapLogFile(int, const char *); +SQUIDCEXTERN char *storeSwapFullPath(int, char *); +SQUIDCEXTERN char *storeSwapSubSubDir(int, char *); +SQUIDCEXTERN const char *storeSwapPath(int); +SQUIDCEXTERN int storeDirWriteCleanLogs(int reopen); +SQUIDCEXTERN STDIRSELECT *storeDirSelectSwapDir; +SQUIDCEXTERN int storeVerifySwapDirs(void); +SQUIDCEXTERN void storeDirCloseSwapLogs(void); +SQUIDCEXTERN void storeDirCloseTmpSwapLog(int dirn); +SQUIDCEXTERN void storeDirDiskFull(sdirno); +SQUIDCEXTERN void storeDirOpenSwapLogs(void); +SQUIDCEXTERN void storeDirSwapLog(const StoreEntry *, int op); +SQUIDCEXTERN void storeDirLRUDelete(StoreEntry *); +SQUIDCEXTERN void storeDirLRUAdd(StoreEntry *); +SQUIDCEXTERN int storeDirGetBlkSize(const char *path, int *blksize); +SQUIDCEXTERN int storeDirGetUFSStats(const char *, int *, int *, int *, int *); + + +class SwapDir : public Store +{ + +public: + SwapDir(char const *aType) : theType (aType), cur_size (0), max_size(0), max_objsize (-1), cleanLog(NULL) { fs.blksize = 1024; + path = NULL; } virtual ~SwapDir(); virtual void reconfigure(int, char *) = 0; char const *type() const; + /* official Store interface functions */ + virtual void diskFull(); + + virtual StoreEntry * get + (const cache_key *); + + virtual void get + (String const, STOREGETCLIENT, void * cbdata); + +virtual size_t maxSize() const { return max_size;} + + virtual size_t minSize() const; + virtual void stat (StoreEntry &anEntry) const; + virtual StoreSearch *search(String const url, HttpRequest *) = 0; + + virtual void updateSize(size_t size, int sign); + + /* migrated from store_dir.cc */ + bool objectSizeIsAcceptable(ssize_t objsize) const; + protected: void parseOptions(int reconfiguring); void dumpOptions(StoreEntry * e) const; @@ -68,7 +155,6 @@ private: public: int cur_size; - int low_size; int max_size; char *path; int index; /* This entry's index into the swapDirs array */ @@ -90,11 +176,11 @@ unsigned int read_only: flags; virtual void init() = 0; /* Initialise the fs */ - virtual void newFileSystem(); /* Create a new fs */ + virtual void create(); /* Create a new fs */ virtual void dump(StoreEntry &)const; /* Dump fs config snippet */ virtual bool doubleCheck(StoreEntry &); /* Double check the obj integrity */ virtual void statfs(StoreEntry &) const; /* Dump fs statistics */ - virtual void maintainfs(); /* Replacement maintainence */ + virtual void maintain(); /* Replacement maintainence */ /* <0 == error. > 1000 == error */ virtual int canStore(StoreEntry const &)const = 0; /* Check if the fs will store an object */ /* These two are notifications */ diff --git a/src/cache_cf.cc b/src/cache_cf.cc index b401984471..a0b1d67a89 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.cc,v 1.461 2004/12/22 16:21:33 serassio Exp $ + * $Id: cache_cf.cc,v 1.462 2005/01/03 16:08:25 robertc Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -42,6 +42,7 @@ #include "ConfigParser.h" #include "ACL.h" #include "StoreFileSystem.h" +#include "Parsing.h" #if SQUID_SNMP #include "snmp.h" @@ -215,24 +216,6 @@ wordlistDup(const wordlist * w) * These functions is the same as atoi/l/f, except that they check for errors */ -long -xatol(const char *token) -{ - char *end; - long ret = strtol(token, &end, 10); - - if (ret == 0 && end == token) - self_destruct(); - - return ret; -} - -int -xatoi(const char *token) -{ - return xatol(token); -} - static double xatof(const char *token) { @@ -245,21 +228,6 @@ xatof(const char *token) return ret; } -int -GetInteger(void) -{ - char *token = strtok(NULL, w_space); - int i; - - if (token == NULL) - self_destruct(); - - if (sscanf(token, "%d", &i) != 1) - self_destruct(); - - return i; -} - static void update_maxobjsize(void) { @@ -267,10 +235,11 @@ update_maxobjsize(void) ssize_t ms = -1; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - assert (Config.cacheSwap.swapDirs[i]); + assert (Config.cacheSwap.swapDirs[i].getRaw()); - if (Config.cacheSwap.swapDirs[i]->max_objsize > ms) - ms = Config.cacheSwap.swapDirs[i]->max_objsize; + if (dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())-> + max_objsize > ms) + ms = dynamic_cast(Config.cacheSwap.swapDirs[i].getRaw())->max_objsize; } store_maxobjsize = ms; @@ -421,13 +390,11 @@ configDoConfigure(void) if (Config.cacheSwap.swapDirs == NULL) fatal("No cache_dir's specified in config file"); - /* calculate Config.Swap.maxSize */ - storeDirConfigure(); - - if (0 == Config.Swap.maxSize) + if (0 == Store::Root().maxSize()) /* people might want a zero-sized cache on purpose */ (void) 0; - else if (Config.Swap.maxSize < (Config.memMaxSize >> 10)) + else if (Store::Root().maxSize() < (Config.memMaxSize >> 10)) + /* This is bogus. folk with NULL caches will want this */ debug(3, 0) ("WARNING cache_mem is larger than total disk cache space!\n"); if (Config.Announce.period > 0) { @@ -1304,7 +1271,7 @@ dump_cachedir(StoreEntry * entry, const char *name, _SquidConfig::_cacheSwap swa assert (entry); for (i = 0; i < swap.n_configured; i++) { - s = swap.swapDirs[i]; + s = dynamic_cast(swap.swapDirs[i].getRaw()); storeAppendPrintf(entry, "%s %s %s", name, s->type(), s->path); s->dump(*entry); storeAppendPrintf(entry, "\n"); @@ -1378,24 +1345,6 @@ dump_authparam(StoreEntry * entry, const char *name, authConfig cfg) (*i)->dump(entry, name, (*i)); } -void -allocate_new_swapdir(_SquidConfig::_cacheSwap * swap) -{ - if (swap->swapDirs == NULL) { - swap->n_allocated = 4; - swap->swapDirs = static_cast(xcalloc(swap->n_allocated, sizeof(SwapDir *))); - } - - if (swap->n_allocated == swap->n_configured) { - SwapDir **tmp; - swap->n_allocated <<= 1; - tmp = static_cast(xcalloc(swap->n_allocated, sizeof(SwapDir *))); - xmemcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *)); - xfree(swap->swapDirs); - swap->swapDirs = tmp; - } -} - /* TODO: just return the object, the # is irrelevant */ static int find_fstype(char *type) @@ -1412,7 +1361,7 @@ parse_cachedir(_SquidConfig::_cacheSwap * swap) { char *type_str; char *path_str; - SwapDir *sd; + RefCount sd; int i; int fs; @@ -1441,9 +1390,18 @@ parse_cachedir(_SquidConfig::_cacheSwap * swap) */ for (i = 0; i < swap->n_configured; i++) { - assert (swap->swapDirs[i]); + assert (swap->swapDirs[i].getRaw()); - if (0 == strcasecmp(path_str, swap->swapDirs[i]->path)) { + /* this is specific to on-fs Stores. The right + * way to handle this is probably to have a mapping + * from paths to stores, and have on-fs stores + * register with that, and lookip in that in their + * own setup logic. RBC 20041225. TODO. + */ + + if (0 == strcasecmp(path_str, dynamic_cast(swap->swapDirs[i].getRaw())-> + path)) { + /* existing configured swap dir */ /* This is a little weird, you'll appreciate it later */ fs = find_fstype(type_str); @@ -1451,9 +1409,14 @@ parse_cachedir(_SquidConfig::_cacheSwap * swap) fatalf("Unknown cache_dir type '%s'\n", type_str); } - sd = swap->swapDirs[i]; + /* TODO: warn here on type changing */ + + sd = dynamic_cast(swap->swapDirs[i].getRaw()); + sd->reconfigure (i, path_str); + update_maxobjsize(); + return; } } @@ -1469,7 +1432,7 @@ parse_cachedir(_SquidConfig::_cacheSwap * swap) allocate_new_swapdir(swap); swap->swapDirs[swap->n_configured] = StoreFileSystem::FileSystems().items[fs]->createSwapDir(); - sd = swap->swapDirs[swap->n_configured]; + sd = dynamic_cast(swap->swapDirs[swap->n_configured].getRaw()); /* parse the FS parameters and options */ sd->parse(swap->n_configured, path_str); ++swap->n_configured; @@ -1477,27 +1440,6 @@ parse_cachedir(_SquidConfig::_cacheSwap * swap) update_maxobjsize(); } -static void -free_cachedir(_SquidConfig::_cacheSwap * swap) -{ - int i; - /* DON'T FREE THESE FOR RECONFIGURE */ - - if (reconfiguring) - return; - - for (i = 0; i < swap->n_configured; i++) { - SwapDir * s = swap->swapDirs[i]; - swap->swapDirs[i] = NULL; - delete s; - } - - safe_free(swap->swapDirs); - swap->swapDirs = NULL; - swap->n_allocated = 0; - swap->n_configured = 0; -} - static const char * peer_type_str(const peer_t type) { @@ -2539,7 +2481,7 @@ free_removalpolicy(RemovalPolicySettings ** settings) free_wordlist(&(*settings)->args); - xfree(*settings); + delete *settings; *settings = NULL; } @@ -2550,7 +2492,7 @@ parse_removalpolicy(RemovalPolicySettings ** settings) if (*settings) free_removalpolicy(settings); - *settings = static_cast(xcalloc(1, sizeof(**settings))); + *settings = new RemovalPolicySettings; parse_string(&(*settings)->type); diff --git a/src/cf.data.pre b/src/cf.data.pre index de6c42e937..57760184ec 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1,6 +1,6 @@ # -# $Id: cf.data.pre,v 1.370 2004/12/24 08:05:29 robertc Exp $ +# $Id: cf.data.pre,v 1.371 2005/01/03 16:08:25 robertc Exp $ # # # SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -1111,6 +1111,10 @@ DOC_START you should not change the coss block size after Squid has written some objects to the cache_dir. + The coss file store has changed from 2.5. Now it uses a file + called 'stripe' in the directory names in the config - and + this will be created by squid -z. + Common options: read-only, this cache_dir is read only. @@ -1124,6 +1128,10 @@ DOC_START Note for coss, max-size must be less than COSS_MEMBUF_SZ, which can be changed with the --with-coss-membuf-size=N configure option. + + The null store type: + + no options are allowed or required DOC_END @@ -3350,7 +3358,7 @@ LOC: Config.Store.objectsPerBucket DOC_START Target number of objects per bucket in the store hash table. Lowering this value increases the total number of buckets and - also the storage maintenance rate. The default is 50. + also the storage maintenance rate. The default is 20. DOC_END NAME: client_db diff --git a/src/defines.h b/src/defines.h index 0699417079..5aa986750c 100644 --- a/src/defines.h +++ b/src/defines.h @@ -1,6 +1,6 @@ /* - * $Id: defines.h,v 1.115 2004/12/20 16:30:35 robertc Exp $ + * $Id: defines.h,v 1.116 2005/01/03 16:08:26 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -203,9 +203,6 @@ #define SwapMetaData(x) &x[STORE_META_TLD_START] #define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(size_t)) -#define STORE_ENTRY_WITH_MEMOBJ 1 -#define STORE_ENTRY_WITHOUT_MEMOBJ 0 - #define PINGER_PAYLOAD_SZ 8192 #define COUNT_INTERVAL 60 diff --git a/src/disk.cc b/src/disk.cc index af51919297..e591c61b3d 100644 --- a/src/disk.cc +++ b/src/disk.cc @@ -1,6 +1,6 @@ /* - * $Id: disk.cc,v 1.168 2004/12/20 16:30:35 robertc Exp $ + * $Id: disk.cc,v 1.169 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -507,3 +507,38 @@ file_read(int fd, char *buf, int req_len, off_t offset, DRCB * handler, void *cl diskHandleRead(fd, ctrl_dat); PROF_stop(file_read); } + +void +safeunlink(const char *s, int quiet) +{ + statCounter.syscalls.disk.unlinks++; + + if (unlink(s) < 0 && !quiet) + debug(50, 1) ("safeunlink: Couldn't delete %s: %s\n", s, xstrerror()); +} + +/* + * Same as rename(2) but complains if something goes wrong; + * the caller is responsible for handing and explaining the + * consequences of errors. + */ +int +xrename(const char *from, const char *to) +{ + debug(21, 2) ("xrename: renaming %s to %s\n", from, to); +#if defined (_SQUID_OS2_) || defined (_SQUID_WIN32_) + + remove + (to); + +#endif + + if (0 == rename(from, to)) + return 0; + + debug(21, errno == ENOENT ? 2 : 1) ("xrename: Cannot rename %s to %s: %s\n", + from, to, xstrerror()); + + return -1; +} + diff --git a/src/fs/coss/CossSwapDir.h b/src/fs/coss/CossSwapDir.h index 31935a8c6f..8f25cc5afc 100644 --- a/src/fs/coss/CossSwapDir.h +++ b/src/fs/coss/CossSwapDir.h @@ -2,6 +2,7 @@ #define __COSSSWAPDIR_H__ #include "SwapDir.h" +#include "StoreSearch.h" #ifndef COSS_MEMBUF_SZ #define COSS_MEMBUF_SZ 1048576 @@ -32,9 +33,10 @@ class CossSwapDir : public SwapDir, public IORequestor public: CossSwapDir(); virtual void init(); - virtual void newFileSystem(); + virtual void create(); virtual void dump(StoreEntry &)const; ~CossSwapDir(); + virtual StoreSearch *search(String const url, HttpRequest *); virtual void unlink (StoreEntry &); virtual void statfs (StoreEntry &)const; virtual int canStore(StoreEntry const &)const; @@ -83,12 +85,43 @@ private: void optionIODump(StoreEntry * e) const; void CossSwapDir::optionBlockSizeDump(StoreEntry *) const; bool CossSwapDir::optionBlockSizeParse(const char *, const char *, int); + char const *stripePath() const; ConfigOption * getOptionTree() const; const char *ioModule; ConfigOptionVector *currentIOOptions; + const char *stripe_path; }; extern void storeCossAdd(CossSwapDir *, StoreEntry *); extern void storeCossRemove(CossSwapDir *, StoreEntry *); extern void storeCossStartMembuf(CossSwapDir * SD); + +class StoreSearchCoss : public StoreSearch +{ + +public: + StoreSearchCoss(RefCount sd); + StoreSearchCoss(StoreSearchCoss const &); + ~StoreSearchCoss(); + /* Iterator API - garh, wrong place */ + /* callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + /* return true if a new StoreEntry is immediately available */ + virtual bool next(); + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + +private: + CBDATA_CLASS2(StoreSearchCoss); + RefCount sd; + void (*callback)(void *cbdata); + void *cbdata; + bool _done; + dlink_node * current; + dlink_node * next_; +}; + #endif diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc index 84948e1fb3..79c28998e3 100644 --- a/src/fs/coss/store_dir_coss.cc +++ b/src/fs/coss/store_dir_coss.cc @@ -1,6 +1,7 @@ /* - * $Id: store_dir_coss.cc,v 1.56 2004/12/20 16:30:42 robertc Exp $ + * $Id: store_dir_coss.cc,v 1.57 2005/01/03 16:08:27 robertc Exp $ + * vim: set et : * * DEBUG: section 47 Store COSS Directory Routines * AUTHOR: Eric Stern @@ -46,6 +47,7 @@ #include "DiskIO/ReadRequest.h" #include "ConfigOption.h" #include "StoreFScoss.h" +#include "Parsing.h" #define STORE_META_BUFSZ 4096 @@ -333,7 +335,7 @@ CossSwapDir::init() io->init(); openLog(); storeCossDirRebuild(this); - theFile = io->newFile(path); + theFile = io->newFile(stripePath()); theFile->open(O_RDWR | O_CREAT, 0644, this); ++n_coss_dirs; @@ -417,7 +419,8 @@ storeCossRebuildFromSwapLog(void *data) } else if (s.op == SWAP_LOG_DEL) { /* Delete unless we already have a newer copy */ - if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) { + if ((e = rb->sd->get + (s.key)) != NULL && s.lastref > e->lastref) { /* * Make sure we don't unlink the file, it might be * in use by a subsequent entry. Also note that @@ -466,7 +469,8 @@ storeCossRebuildFromSwapLog(void *data) continue; } - e = storeGet(s.key); + e = rb->sd->get + (s.key); if (e) { /* key already exists, current entry is newer */ @@ -513,7 +517,7 @@ storeCossAddDiskRestore(CossSwapDir * SD, const cache_key * key, debug(47, 5) ("storeCossAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); /* if you call this you'd better be sure file_number is not * already in use! */ - e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e = new StoreEntry(); e->store_status = STORE_OK; e->swap_dirn = SD->index; storeSetMemStatus(e, NOT_IN_MEMORY); @@ -546,12 +550,10 @@ storeCossDirRebuild(CossSwapDir * sd) int clean = 0; int zero = 0; FILE *fp; - EVH *func = NULL; CBDATA_INIT_TYPE(RebuildState); rb = cbdataAlloc(RebuildState); rb->sd = sd; rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; - func = storeCossRebuildFromSwapLog; rb->flags.clean = (unsigned int) clean; /* * If the swap.state file exists in the cache_dir, then @@ -584,7 +586,7 @@ storeCossDirRebuild(CossSwapDir * sd) return; } - eventAdd("storeCossRebuild", func, rb, 0.0, 1); + eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); } static void @@ -625,7 +627,7 @@ storeCossDirOpenTmpSwapLog(CossSwapDir * sd, int *clean_flag, int *zero_flag) FILE *fp; int anfd; - if (stat(swaplog_path, &log_sb) < 0) { + if (::stat(swaplog_path, &log_sb) < 0) { debug(50, 1) ("Cache COSS Dir #%d: No log file\n", sd->index); safe_free(swaplog_path); safe_free(clean_path); @@ -658,7 +660,7 @@ storeCossDirOpenTmpSwapLog(CossSwapDir * sd, int *clean_flag, int *zero_flag) memset(&clean_sb, '\0', sizeof(struct stat)); - if (stat(clean_path, &clean_sb) < 0) + if (::stat(clean_path, &clean_sb) < 0) *clean_flag = 0; else if (clean_sb.st_mtime < log_sb.st_mtime) *clean_flag = 0; @@ -733,7 +735,7 @@ CossSwapDir::writeCleanStart() state->newLog, state->fd); #if HAVE_FCHMOD - if (stat(state->cur, &sb) == 0) + if (::stat(state->cur, &sb) == 0) fchmod(state->fd, sb.st_mode); #endif @@ -743,6 +745,10 @@ CossSwapDir::writeCleanStart() return 0; } +/* RBC 20050101 - I think there is a race condition here, + * *current can be freed as its not ref counted, if/when + * the store overruns the log writer + */ const StoreEntry * CossCleanLog::nextEntry() { @@ -895,22 +901,56 @@ CossSwapDir::logEntry(const StoreEntry & e, int op) const } void -CossSwapDir::newFileSystem() +CossSwapDir::create() { - debug(47, 3) ("Creating swap space in %s\n", path); - debug (47,0)("COSS autocreation is not implemented. Please create the file manually\n"); + debugs (47, 3, "Creating swap space in " << path); + + struct stat swap_sb; + int swap; + + if (::stat(path, &swap_sb) < 0) { + debugs (47, 2, "COSS swap space space being allocated."); + mkdir(path, 0700); + } + + /* should check here for directories instead of files, and for file size + * TODO - if nothing changes, there is nothing to do + */ + swap = open(stripePath(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0600); + + /* TODO just set the file size */ + /* swap size is in K */ + char *block[1024]; + + memset(&block, '\0', 1024); + + for (off_t offset = 0; offset < max_size; ++offset) { + if (write (swap, block, 1024) < 1024) { + debugs (47, 0, "Failed to create COSS swap space in " << path); + } + } + + close (swap); + } /* we are shutting down, flush all membufs to disk */ CossSwapDir::~CossSwapDir() { io->sync(); - theFile->close(); + + if (theFile != NULL) + theFile->close(); + delete io; closeLog(); + n_coss_dirs--; + safe_free(ioModule); + + safe_free(stripe_path); } /* @@ -1050,13 +1090,14 @@ CossSwapDir::dump(StoreEntry &entry)const dumpOptions(&entry); } -CossSwapDir::CossSwapDir() : SwapDir ("coss"), swaplog_fd(-1), count(0), current_membuf (NULL), current_offset(0), numcollisions(0), blksz_bits(0), io (NULL), ioModule(NULL), currentIOOptions(new ConfigOptionVector()) +CossSwapDir::CossSwapDir() : SwapDir ("coss"), swaplog_fd(-1), count(0), current_membuf (NULL), current_offset(0), numcollisions(0), blksz_bits(0), io (NULL), ioModule(NULL), currentIOOptions(new ConfigOptionVector()), stripe_path(NULL) { membufs.head = NULL; membufs.tail = NULL; cossindex.head = NULL; cossindex.tail = NULL; blksz_mask = (1 << blksz_bits) - 1; + repl = NULL; } bool @@ -1110,3 +1151,80 @@ CossSwapDir::optionBlockSizeDump(StoreEntry * e) const { storeAppendPrintf(e, " block-size=%d", 1 << blksz_bits); } + +StoreSearch * +CossSwapDir::search(String const url, HttpRequest *) +{ + if (url.size()) + fatal ("Cannot search by url yet\n"); + + return new StoreSearchCoss (this); +} + +char const * +CossSwapDir::stripePath() const +{ + if (!stripe_path) { + String result = path; + result.append("/stripe"); + const_cast(this)->stripe_path = xstrdup(result.buf()); + } + + return stripe_path; +} + +CBDATA_CLASS_INIT(StoreSearchCoss); +StoreSearchCoss::StoreSearchCoss(RefCount aSwapDir) : sd(aSwapDir), callback (NULL), cbdata(NULL), _done (false), current(NULL), next_(sd->cossindex.tail) +{ + /* TODO: this races with the store as does the cleanlog stuff. + * FIXME by making coss_nodes ref counted */ +} + +/* do not link +StoreSearchCoss::StoreSearchCoss(StoreSearchCoss const &); +*/ + +StoreSearchCoss::~StoreSearchCoss() +{} + +void +StoreSearchCoss::next(void (callback)(void *cbdata), void *cbdata) +{ + next(); + callback (cbdata); +} + +bool +StoreSearchCoss::next() +{ + current = next_; + + if (next_) + next_ = next_->prev; + + if (!current) + _done = true; + + return current != NULL; +} + +bool +StoreSearchCoss::error() const +{ + return false; +} + +bool +StoreSearchCoss::isDone() const +{ + return _done; +} + +StoreEntry * +StoreSearchCoss::currentItem() +{ + if (!current) + return NULL; + + return static_cast( current->data ); +} diff --git a/src/fs/null/store_null.cc b/src/fs/null/store_null.cc index 79a2d562ee..cbaa87de64 100644 --- a/src/fs/null/store_null.cc +++ b/src/fs/null/store_null.cc @@ -1,6 +1,6 @@ /* - * $Id: store_null.cc,v 1.7 2003/07/22 15:23:14 robertc Exp $ + * $Id: store_null.cc,v 1.8 2005/01/03 16:08:27 robertc Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -98,3 +98,53 @@ NullSwapDir::parse(int anIndex, char *aPath) parseOptions(0); } +StoreSearch * +NullSwapDir::search(String const url, HttpRequest *) +{ + if (url.size()) + fatal ("Cannot search by url yet\n"); + + return new StoreSearchNull (); +} + + +CBDATA_CLASS_INIT(StoreSearchNull); +StoreSearchNull::StoreSearchNull() +{} + +/* do not link +StoreSearchNull::StoreSearchNull(StoreSearchNull const &); +*/ + +StoreSearchNull::~StoreSearchNull() +{} + +void +StoreSearchNull::next(void (callback)(void *cbdata), void *cbdata) +{ + callback (cbdata); +} + +bool +StoreSearchNull::next() +{ + return false; +} + +bool +StoreSearchNull::error() const +{ + return false; +} + +bool +StoreSearchNull::isDone() const +{ + return true; +} + +StoreEntry * +StoreSearchNull::currentItem() +{ + return NULL; +} diff --git a/src/fs/null/store_null.h b/src/fs/null/store_null.h index 521f6828e2..5dbde9a047 100644 --- a/src/fs/null/store_null.h +++ b/src/fs/null/store_null.h @@ -1,6 +1,6 @@ /* - * $Id: store_null.h,v 1.1 2003/07/22 15:23:14 robertc Exp $ + * $Id: store_null.h,v 1.2 2005/01/03 16:08:27 robertc Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -36,6 +36,7 @@ #include "squid.h" #include "SwapDir.h" +#include "StoreSearch.h" class NullSwapDir : public SwapDir { @@ -48,6 +49,29 @@ public: virtual StoreIOState::Pointer openStoreIO(StoreEntry &, STFNCB *, STIOCB *, void *); virtual void parse(int, char*); virtual void reconfigure (int, char *); + virtual StoreSearch *search(String const url, HttpRequest *); +}; + +class StoreSearchNull : public StoreSearch +{ + +public: + StoreSearchNull(); + StoreSearchNull(StoreSearchNull const &); + ~StoreSearchNull(); + /* Iterator API - garh, wrong place */ + /* callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + /* return true if a new StoreEntry is immediately available */ + virtual bool next(); + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + +private: + CBDATA_CLASS2(StoreSearchNull); }; #endif /* SQUID_STORE_NULL_H */ diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/store_dir_ufs.cc index b2a4e314a1..30eb81b7c9 100644 --- a/src/fs/ufs/store_dir_ufs.cc +++ b/src/fs/ufs/store_dir_ufs.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir_ufs.cc,v 1.67 2004/12/20 16:30:45 robertc Exp $ + * $Id: store_dir_ufs.cc,v 1.68 2005/01/03 16:08:27 robertc Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -41,6 +41,7 @@ #include "ConfigOption.h" #include "DiskIO/DiskIOStrategy.h" #include "DiskIO/DiskIOModule.h" +#include "Parsing.h" #include "SwapDir.h" int UFSSwapDir::NumberOfUFSDirs = 0; @@ -230,7 +231,7 @@ UFSSwapDir::init() } void -UFSSwapDir::newFileSystem() +UFSSwapDir::create() { debug(47, 3) ("Creating swap space in %s\n", path); createDirectory(path, 0); @@ -276,7 +277,7 @@ UFSSwapDir::doubleCheck(StoreEntry & e) struct stat sb; - if (stat(fullPath(e.swap_filen, NULL), &sb) < 0) { + if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { debug(47, 0) ("UFSSwapDir::doubleCheck: MISSING SWAP FILE\n"); dumpEntry(e); return true; @@ -337,7 +338,7 @@ UFSSwapDir::statfs(StoreEntry & sentry) const } void -UFSSwapDir::maintainfs() +UFSSwapDir::maintain() { /* We can't delete objects while rebuilding swap */ @@ -350,7 +351,7 @@ UFSSwapDir::maintainfs() RemovalPurgeWalker *walker; - double f = (double) (cur_size - low_size) / (max_size - low_size); + double f = (double) (cur_size - minSize()) / (max_size - minSize()); f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; @@ -368,7 +369,7 @@ UFSSwapDir::maintainfs() walker = repl->PurgeInit(repl, max_scan); while (1) { - if (cur_size < low_size) + if (cur_size < (int) minSize()) /* cur_size should be unsigned */ break; if (removed >= max_remove) @@ -385,7 +386,7 @@ UFSSwapDir::maintainfs() } walker->Done(walker); - debug(47, (removed ? 2 : 3)) ("UFSSwapDir::maintainfs: %s removed %d/%d f=%.03f max_scan=%d\n", + debug(47, (removed ? 2 : 3)) ("UFSSwapDir::maintain: %s removed %d/%d f=%.03f max_scan=%d\n", path, removed, max_remove, f, max_scan); } @@ -470,7 +471,7 @@ UFSSwapDir::mapBitAllocate() } /* - * Initialise the asyncufs bitmap + * Initialise the ufs bitmap * * If there already is a bitmap, and the numobjects is larger than currently * configured, we allocate a new bitmap and 'grow' the old one into it. @@ -506,7 +507,7 @@ UFSSwapDir::createDirectory(const char *path, int should_exist) struct stat st; getCurrentTime(); - if (0 == stat(path, &st)) { + if (0 == ::stat(path, &st)) { if (S_ISDIR(st.st_mode)) { debug(47, should_exist ? 3 : 1) ("%s exists\n", path); } else { @@ -536,7 +537,7 @@ UFSSwapDir::pathIsDirectory(const char *path)const struct stat sb; - if (stat(path, &sb) < 0) { + if (::stat(path, &sb) < 0) { debug(47, 0) ("%s: %s\n", path, xstrerror()); return false; } @@ -706,7 +707,7 @@ UFSSwapDir::addDiskRestore(const cache_key * key, debug(47, 5) ("commonUfsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); /* if you call this you'd better be sure file_number is not * already in use! */ - e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + e = new StoreEntry(); e->store_status = STORE_OK; storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_DONE; @@ -734,41 +735,8 @@ UFSSwapDir::addDiskRestore(const cache_key * key, void UFSSwapDir::rebuild() { - int clean = 0; - int zeroLengthLog = 0; - FILE *fp; - EVH *func = NULL; - RebuildState *rb = new RebuildState; - rb->sd = this; - rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; - /* - * If the swap.state file exists in the cache_dir, then - * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll - * use commonUfsDirRebuildFromDirectory() to open up each file - * and suck in the meta data. - */ - fp = openTmpSwapLog(&clean, &zeroLengthLog); - - if (fp == NULL || zeroLengthLog) { - if (fp != NULL) - fclose(fp); - - func = RebuildState::RebuildFromDirectory; - } else { - func = RebuildState::RebuildFromSwapLog; - rb->log = fp; - rb->flags.clean = (unsigned int) clean; - } - - if (!clean) - rb->flags.need_to_validate = 1; - - debug(47, 1) ("Rebuilding storage in %s (%s)\n", - path, clean ? "CLEAN" : "DIRTY"); - store_dirs_rebuilding++; - - eventAdd("storeRebuild", func, rb, 0.0, 1); + eventAdd("storeRebuild", RebuildState::RebuildStep, new RebuildState(this), 0.0, 1); } void @@ -809,7 +777,7 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) FILE *fp; int fd; - if (stat(swaplog_path, &log_sb) < 0) { + if (::stat(swaplog_path, &log_sb) < 0) { debug(47, 1) ("Cache Dir #%d: No log file\n", index); safe_free(swaplog_path); safe_free(clean_path); @@ -842,7 +810,7 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) memset(&clean_sb, '\0', sizeof(struct stat)); - if (stat(clean_path, &clean_sb) < 0) + if (::stat(clean_path, &clean_sb) < 0) *clean_flag = 0; else if (clean_sb.st_mtime < log_sb.st_mtime) *clean_flag = 0; @@ -918,7 +886,7 @@ UFSSwapDir::writeCleanStart() state->newLog, state->fd); #if HAVE_FCHMOD - if (stat(state->cur, &sb) == 0) + if (::stat(state->cur, &sb) == 0) fchmod(state->fd, sb.st_mode); #endif @@ -1156,7 +1124,7 @@ UFSSwapDir::DirClean(int swap_index) #if USE_TRUNCATE - if (!stat(de->d_name, &sb)) + if (!::stat(de->d_name, &sb)) if (sb.st_size == 0) continue; @@ -1215,7 +1183,8 @@ UFSSwapDir::CleanEvent(void *unused) UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { - sd = INDEXSD(i); + /* This is bogus, the controller should just clean each instance once */ + sd = dynamic_cast (INDEXSD(i)); if (!UFSSwapDir::IsUFSDir(sd)) continue; @@ -1268,7 +1237,7 @@ UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) int L1, L2; int filn = fn; assert(F0 < Config.cacheSwap.n_configured); - assert (UFSSwapDir::IsUFSDir (INDEXSD(F0))); + assert (UFSSwapDir::IsUFSDir (dynamic_cast(INDEXSD(F0)))); UFSSwapDir *sd = dynamic_cast(INDEXSD(F0)); if (!sd) @@ -1351,14 +1320,14 @@ UFSSwapDir::replacementAdd(StoreEntry * e) void UFSSwapDir::replacementRemove(StoreEntry * e) { - SwapDir *SD; + StorePointer SD; if (e->swap_dirn < 0) return; SD = INDEXSD(e->swap_dirn); - assert (dynamic_cast(SD) == this); + assert (dynamic_cast(SD.getRaw()) == this); debug(47, 4) ("UFSSwapDir::replacementRemove: remove node %p from dir %d\n", e, index); @@ -1408,3 +1377,67 @@ UFSSwapDir::sync() { IO->sync(); } + +StoreSearch * +UFSSwapDir::search(String const url, HttpRequest *request) +{ + if (url.size()) + fatal ("Cannot search by url yet\n"); + + return new StoreSearchUFS (this); +} + +CBDATA_CLASS_INIT(StoreSearchUFS); +StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), current (NULL), _done (false) +{} + +/* do not link +StoreSearchUFS::StoreSearchUFS(StoreSearchUFS const &); +*/ + +StoreSearchUFS::~StoreSearchUFS() +{ + walker->Done(walker); + walker = NULL; +} + +void +StoreSearchUFS::next(void (callback)(void *cbdata), void *cbdata) +{ + next(); + callback (cbdata); +} + +bool +StoreSearchUFS::next() +{ + /* the walker API doesn't make sense. the store entries referred to are already readwrite + * from their hash table entries + */ + + if (walker) + current = const_cast(walker->Next(walker)); + + if (current == NULL) + _done = true; + + return current != NULL; +} + +bool +StoreSearchUFS::error() const +{ + return false; +} + +bool +StoreSearchUFS::isDone() const +{ + return _done; +} + +StoreEntry * +StoreSearchUFS::currentItem() +{ + return current; +} diff --git a/src/fs/ufs/ufscommon.cc b/src/fs/ufs/ufscommon.cc index 0fb895e82b..075901823c 100644 --- a/src/fs/ufs/ufscommon.cc +++ b/src/fs/ufs/ufscommon.cc @@ -1,5 +1,6 @@ /* - * $Id: ufscommon.cc,v 1.2 2004/12/21 17:28:30 robertc Exp $ + * $Id: ufscommon.cc,v 1.3 2005/01/03 16:08:27 robertc Exp $ + * vim: set et : * * DEBUG: section 47 Store Directory Routines * AUTHOR: Robert Collins @@ -44,34 +45,65 @@ CBDATA_CLASS_INIT(RebuildState); -void * -RebuildState::operator new (size_t size) +RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir), e(NULL), fromLog(true), _done (false) { - assert (size == sizeof(RebuildState)); - CBDATA_INIT_TYPE(RebuildState); - RebuildState *result = cbdataAlloc(RebuildState); - return result; -} + speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll + * use commonUfsDirRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + int clean = 0; + int zeroLengthLog = 0; + FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); + + if (fp == NULL || zeroLengthLog) { + fromLog = false; + + if (fp != NULL) + fclose(fp); + + } else { + fromLog = true; + log = fp; + flags.clean = (unsigned int) clean; + } -void -RebuildState::operator delete (void *address) -{ - RebuildState *t = static_cast(address); - cbdataFree(t); + if (!clean) + flags.need_to_validate = 1; + + debug(47, 1) ("Rebuilding storage in %s (%s)\n", + sd->path, clean ? "CLEAN" : "DIRTY"); } RebuildState::~RebuildState() { - store_dirs_rebuilding--; sd->closeTmpSwapLog(); - storeRebuildComplete(&counts); } void -RebuildState::RebuildFromDirectory(void *data) +RebuildState::RebuildStep(void *data) { RebuildState *rb = (RebuildState *)data; - rb->rebuildFromDirectory(); + rb->rebuildStep(); + + if (!rb->isDone()) + eventAdd("storeRebuild", RebuildStep, rb, 0.0, 1); + else { + store_dirs_rebuilding--; + storeRebuildComplete(&rb->counts); + delete rb; + } +} + +void +RebuildState::rebuildStep() +{ + if (fromLog) + rebuildFromSwapLog(); + else + rebuildFromDirectory(); } struct InitStoreEntry : public unary_function @@ -125,7 +157,7 @@ RebuildState::rebuildFromDirectory() if (fd == -2) { debug(47, 1) ("Done scanning %s swaplog (%d entries)\n", sd->path, n_read); - delete this; + _done = true; return; } else if (fd < 0) { continue; @@ -225,7 +257,25 @@ RebuildState::rebuildFromDirectory() continue; } - e = storeGet(key); + /* this needs to become + * 1) unpack url + * 2) make synthetic request with headers ?? or otherwise search + * for a matching object in the store + * TODO FIXME change to new async api + * TODO FIXME I think there is a race condition here with the + * async api : + * store A reads in object foo, searchs for it, and finds nothing. + * store B reads in object foo, searchs for it, finds nothing. + * store A gets called back with nothing, so registers the object + * store B gets called back with nothing, so registers the object, + * which will conflict when the in core index gets around to scanning + * store B. + * + * this suggests that rather than searching for duplicates, the + * index rebuild should just assume its the most recent accurate + * store entry and whoever indexes the stores handles duplicates. + */ + e = Store::Root().get(key); if (e && e->lastref >= tmpe.lastref) { /* key already exists, current entry is newer */ @@ -241,27 +291,19 @@ RebuildState::rebuildFromDirectory() counts.objcount++; storeEntryDump(&tmpe, 5); - e = sd->addDiskRestore(key, - filn, - tmpe.swap_file_sz, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastmod, - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ - (int) flags.clean); - storeDirSwapLog(e, SWAP_LOG_ADD); + currentEntry(sd->addDiskRestore(key, + filn, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ + (int) flags.clean)); + storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); } - eventAdd("storeRebuild", RebuildFromDirectory, this, 0.0, 1); -} - -void -RebuildState::RebuildFromSwapLog(void *data) -{ - RebuildState *rb = (RebuildState *)data; - rb->rebuildFromSwapLog(); } StoreEntry * @@ -284,24 +326,24 @@ RebuildState::rebuildFromSwapLog() /* load a number of objects per invocation */ for (int count = 0; count < speed; count++) { - StoreSwapLogData s; + StoreSwapLogData swapData; size_t ss = sizeof(StoreSwapLogData); - if (fread(&s, ss, 1, log) != 1) { + if (fread(&swapData, ss, 1, log) != 1) { debug(47, 1) ("Done reading %s swaplog (%d entries)\n", sd->path, n_read); fclose(log); log = NULL; - delete this; + _done = true; return; } n_read++; - if (s.op <= SWAP_LOG_NOP) + if (swapData.op <= SWAP_LOG_NOP) continue; - if (s.op >= SWAP_LOG_MAX) + if (swapData.op >= SWAP_LOG_MAX) continue; /* @@ -312,20 +354,26 @@ RebuildState::rebuildFromSwapLog() * bits. Now, for backwards compatibility, we just need * to mask it off. */ - s.swap_filen &= 0x00FFFFFF; + swapData.swap_filen &= 0x00FFFFFF; debug(47, 3) ("commonUfsDirRebuildFromSwapLog: %s %s %08X\n", - swap_log_op_str[(int) s.op], - storeKeyText(s.key), - s.swap_filen); + swap_log_op_str[(int) swapData.op], + storeKeyText(swapData.key), + swapData.swap_filen); - if (s.op == SWAP_LOG_ADD) { + if (swapData.op == SWAP_LOG_ADD) { (void) 0; - } else if (s.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy */ - currentEntry (storeGet(s.key)); - - if (currentEntry() != NULL && s.lastref > e->lastref) { + } else if (swapData.op == SWAP_LOG_DEL) { + /* Delete unless we already have a newer copy anywhere in any store */ + /* this needs to become + * 1) unpack url + * 2) make synthetic request with headers ?? or otherwise search + * for a matching object in the store + * TODO FIXME change to new async api + */ + currentEntry (Store::Root().get(swapData.key)); + + if (currentEntry() != NULL && swapData.lastref > e->lastref) { /* * Make sure we don't unlink the file, it might be * in use by a subsequent entry. Also note that @@ -372,39 +420,48 @@ RebuildState::rebuildFromSwapLog() (int) sb.st_size / ss, n_read); } - if (!sd->validFileno(s.swap_filen, 0)) { + if (!sd->validFileno(swapData.swap_filen, 0)) { counts.invalid++; continue; } - if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) { counts.badflags++; continue; } - currentEntry(storeGet(s.key)); + /* this needs to become + * 1) unpack url + * 2) make synthetic request with headers ?? or otherwise search + * for a matching object in the store + * TODO FIXME change to new async api + */ + currentEntry (Store::Root().get(swapData.key)); + int used; /* is swapfile already in use? */ - used = sd->mapBitTest(s.swap_filen); + + used = sd->mapBitTest(swapData.swap_filen); + /* If this URL already exists in the cache, does the swap log * appear to have a newer entry? Compare 'lastref' from the * swap log to e->lastref. */ /* is the log entry newer than current entry? */ - int disk_entry_newer = currentEntry() ? (s.lastref > currentEntry()->lastref ? 1 : 0) : 0; + int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; if (used && !disk_entry_newer) { /* log entry is old, ignore it */ counts.clashcount++; continue; - } else if (used && currentEntry() && currentEntry()->swap_filen == s.swap_filen && currentEntry()->swap_dirn == sd->index) { + } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { /* swapfile taken, same URL, newer, update meta */ if (currentEntry()->store_status == STORE_OK) { - currentEntry()->lastref = s.timestamp; - currentEntry()->timestamp = s.timestamp; - currentEntry()->expires = s.expires; - currentEntry()->lastmod = s.lastmod; - currentEntry()->flags = s.flags; - currentEntry()->refcount += s.refcount; + currentEntry()->lastref = swapData.timestamp; + currentEntry()->timestamp = swapData.timestamp; + currentEntry()->expires = swapData.expires; + currentEntry()->lastmod = swapData.lastmod; + currentEntry()->flags = swapData.flags; + currentEntry()->refcount += swapData.refcount; sd->dereference(*currentEntry()); } else { debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); @@ -419,7 +476,7 @@ RebuildState::rebuildFromSwapLog() * caught this. If the log is clean, there should never be a * newer entry. */ debug(47, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n", - sd->index, s.swap_filen); + sd->index, swapData.swap_filen); /* I'm tempted to remove the swapfile here just to be safe, * but there is a bad race condition in the NOVM version if * the swapfile has recently been opened for writing, but @@ -463,21 +520,20 @@ RebuildState::rebuildFromSwapLog() /* update store_swap_size */ counts.objcount++; - currentEntry(sd->addDiskRestore(s.key, - s.swap_filen, - s.swap_file_sz, - s.expires, - s.timestamp, - s.lastref, - s.lastmod, - s.refcount, - s.flags, + currentEntry(sd->addDiskRestore(swapData.key, + swapData.swap_filen, + swapData.swap_file_sz, + swapData.expires, + swapData.timestamp, + swapData.lastref, + swapData.lastmod, + swapData.refcount, + swapData.flags, (int) flags.clean)); storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); } - eventAdd("storeRebuild", RebuildFromSwapLog, this, 0.0, 1); } int @@ -589,6 +645,43 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) return fd; } +void +RebuildState::next(void (callback)(void *cbdata), void *cbdata) +{ + /* for now, we don't cache at all */ + speed = 1; + currentEntry(NULL); + + while (!isDone() && currentEntry() == NULL) + rebuildStep(); + + callback(cbdata); +} + +bool +RebuildState::next() +{ + return false; +} + +bool +RebuildState::error() const +{ + return false; +} + +bool +RebuildState::isDone() const +{ + return _done; +} + +StoreEntry * +RebuildState::currentItem() +{ + return currentEntry(); +} + #ifndef _USE_INLINE_ #include "ufscommon.cci" #endif diff --git a/src/fs/ufs/ufscommon.cci b/src/fs/ufs/ufscommon.cci index 436398f225..6cc3eff611 100644 --- a/src/fs/ufs/ufscommon.cci +++ b/src/fs/ufs/ufscommon.cci @@ -1,5 +1,5 @@ /* - * $Id: ufscommon.cci,v 1.1 2004/12/20 16:30:45 robertc Exp $ + * $Id: ufscommon.cci,v 1.2 2005/01/03 16:08:27 robertc Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -32,5 +32,3 @@ * */ -RebuildState::RebuildState () : e(NULL){} - diff --git a/src/fs/ufs/ufscommon.h b/src/fs/ufs/ufscommon.h index 2fd1e3d650..06830516bd 100644 --- a/src/fs/ufs/ufscommon.h +++ b/src/fs/ufs/ufscommon.h @@ -1,6 +1,6 @@ /* - * $Id: ufscommon.h,v 1.1 2004/12/20 16:30:45 robertc Exp $ + * $Id: ufscommon.h,v 1.2 2005/01/03 16:08:27 robertc Exp $ * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -41,6 +41,7 @@ /* Common UFS routines */ #include "SwapDir.h" +#include "StoreSearch.h" class UFSStrategy; @@ -58,13 +59,14 @@ public: UFSSwapDir(char const *aType, const char *aModuleType); virtual void init(); - virtual void newFileSystem(); + virtual void create(); virtual void dump(StoreEntry &) const; ~UFSSwapDir(); + virtual StoreSearch *search(String const url, HttpRequest *); virtual bool doubleCheck(StoreEntry &); virtual void unlink(StoreEntry &); virtual void statfs(StoreEntry &)const; - virtual void maintainfs(); + virtual void maintain(); virtual int canStore(StoreEntry const &)const; virtual void reference(StoreEntry &); virtual void dereference(StoreEntry &); @@ -269,18 +271,56 @@ private: MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_read) MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_write) +class StoreSearchUFS : public StoreSearch +{ + +public: + StoreSearchUFS(RefCount sd); + StoreSearchUFS(StoreSearchUFS const &); + virtual ~StoreSearchUFS(); + /* Iterator API - garh, wrong place */ + /* callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + /* return true if a new StoreEntry is immediately available */ + virtual bool next(); + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + RemovalPolicyWalker *walker; + +private: + CBDATA_CLASS2(StoreSearchUFS); + void (callback)(void *cbdata); + void *cbdata; + StoreEntry * current; + bool _done; +}; + class RebuildState : public RefCountable { public: - void *operator new(size_t); - void operator delete(void *); - static EVH RebuildFromDirectory; - static EVH RebuildFromSwapLog; + static EVH RebuildStep; - _SQUID_INLINE_ RebuildState(); + RebuildState(RefCount sd); ~RebuildState(); - UFSSwapDir *sd; + + /* Iterator API - garh, wrong place */ + /* callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + /* return true if a new StoreEntry is immediately available */ + virtual bool next(); + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; int n_read; FILE *log; int speed; @@ -301,8 +341,8 @@ unsigned int init: } flags; - int done; int in_dir; + int done; int fn; struct dirent *entry; @@ -313,13 +353,18 @@ unsigned int init: struct _store_rebuild_data counts; private: - CBDATA_CLASS(RebuildState); + CBDATA_CLASS2(RebuildState); void rebuildFromDirectory(); void rebuildFromSwapLog(); + void rebuildStep(); int getNextFile(sfileno *, int *size); StoreEntry *currentEntry() const; void currentEntry(StoreEntry *); StoreEntry *e; + bool fromLog; + bool _done; + void (callback)(void *cbdata); + void *cbdata; }; #ifdef _USE_INLINE_ diff --git a/src/http.cc b/src/http.cc index 3b817e5157..de58e352e5 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.441 2004/12/21 17:52:53 robertc Exp $ + * $Id: http.cc,v 1.442 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -659,7 +659,7 @@ HttpStateData::processReplyHeader(const char *buf, int size) hdr_len = reply_hdr.size; if (hdr_len > 4 && strncmp(reply_hdr.buf, "HTTP/", 5)) { - debugs(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '" << reply_hdr.buf << "'\n"); + debugs(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '" << reply_hdr.buf << "'"); reply_hdr_state += 2; memBufClean(&reply_hdr); failReply (reply, HTTP_INVALID_HEADER); @@ -673,7 +673,7 @@ HttpStateData::processReplyHeader(const char *buf, int size) hdr_len = hdr_size; if (hdr_len > Config.maxReplyHeaderSize) { - debugs(11, 1, "httpProcessReplyHeader: Too large reply header\n"); + debugs(11, 1, "httpProcessReplyHeader: Too large reply header"); if (!memBufIsNull(&reply_hdr)) memBufClean(&reply_hdr); @@ -712,7 +712,7 @@ HttpStateData::processReplyHeader(const char *buf, int size) httpReplyParse(reply, reply_hdr.buf, hdr_size); if (reply->sline.status >= HTTP_INVALID_HEADER) { - debugs(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '" << reply_hdr.buf << "'\n"); + debugs(11, 3, "httpProcessReplyHeader: Non-HTTP-compliant header: '" << reply_hdr.buf << "'"); failReply (reply, HTTP_INVALID_HEADER); return; } diff --git a/src/list.cc b/src/list.cc new file mode 100644 index 0000000000..aa005fc1f7 --- /dev/null +++ b/src/list.cc @@ -0,0 +1,72 @@ + +/* + * $Id: list.cc,v 1.1 2005/01/03 16:08:26 robertc Exp $ + * + * DEBUG: section ?? Linked list functions + * AUTHOR: Harvest Derived + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +/* This should go away, in favour of the List template class */ + +void +linklistPush(link_list ** L, void *p) +{ + link_list *l = (link_list *)memAllocate(MEM_LINK_LIST); + l->next = NULL; + l->ptr = p; + + while (*L) + L = &(*L)->next; + + *L = l; +} + +void * +linklistShift(link_list ** L) +{ + void *p; + link_list *l; + + if (NULL == *L) + return NULL; + + l = *L; + + p = l->ptr; + + *L = (*L)->next; + + memFree(l, MEM_LINK_LIST); + + return p; +} + diff --git a/src/main.cc b/src/main.cc index acc2fbe540..5e52cb93e2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,6 @@ /* - * $Id: main.cc,v 1.401 2004/12/31 10:29:28 serassio Exp $ + * $Id: main.cc,v 1.402 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -46,6 +46,7 @@ #include "StoreFileSystem.h" #include "DiskIO/DiskIOModule.h" #include "comm.h" +#include "SwapDir.h" #if USE_WIN32_SERVICE @@ -823,7 +824,7 @@ mainInitialize(void) debug(1, 1) ("Ready to serve requests.\n"); if (!configured_once) { - eventAdd("storeMaintain", storeMaintainSwapSpace, NULL, 1.0, 1); + eventAdd("storeMaintain", Store::Maintain, NULL, 1.0, 1); if (Config.onoff.announce) eventAdd("start_announce", start_announce, NULL, 3600.0, 1); @@ -994,6 +995,9 @@ main(int argc, char **argv) /* Shouldn't be needed for config parsing, but have not audited for such */ StoreFileSystem::SetupAllFs(); + /* we may want the parsing process to set this up in the future */ + Store::Root(new StoreController); + parse_err = parseConfigFile(ConfigFile); if (opt_parse_cfg_only) @@ -1047,7 +1051,7 @@ main(int argc, char **argv) setEffectiveUser(); debug(0, 0) ("Creating Swap Directories\n"); - storeCreateSwapDirectories(); + Store::Root().create(); #if USE_WIN32_SERVICE return; @@ -1129,7 +1133,7 @@ main(int argc, char **argv) loop_delay = 0; /* Attempt any pending storedir IO */ - storeDirCallback(); + Store::Root().callback(); comm_calliocallback(); @@ -1488,11 +1492,11 @@ SquidShutdown(void *unused) WIN32_svcstatusupdate(SERVICE_STOP_PENDING, 10000); #endif - storeDirSync(); /* Flush pending object writes/unlinks */ + Store::Root().sync(); /* Flush pending object writes/unlinks */ storeDirWriteCleanLogs(0); PrintRusage(); dumpMallocStats(); - storeDirSync(); /* Flush log writes */ + Store::Root().sync(); /* Flush log writes */ storeLogClose(); accessLogClose(); useragentLogClose(); @@ -1502,7 +1506,7 @@ SquidShutdown(void *unused) fwdUninit(); #endif - storeDirSync(); /* Flush log close */ + Store::Root().sync(); /* Flush log close */ StoreFileSystem::FreeAllFs(); DiskIOModule::FreeAllModules(); #if PURIFY || XMALLOC_TRACE diff --git a/src/neighbors.cc b/src/neighbors.cc index dac8c3dfdd..e257ceaf37 100644 --- a/src/neighbors.cc +++ b/src/neighbors.cc @@ -1,6 +1,6 @@ /* - * $Id: neighbors.cc,v 1.325 2004/04/03 14:07:39 hno Exp $ + * $Id: neighbors.cc,v 1.326 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 15 Neighbor Routines * AUTHOR: Harvest Derived @@ -1061,7 +1061,7 @@ neighborsUdpAck(const cache_key * key, icp_common_t * header, const struct socka debug(15, 6) ("neighborsUdpAck: opcode %d '%s'\n", (int) opcode, storeKeyText(key)); - if (NULL != (entry = storeGet(key))) + if (NULL != (entry = Store::Root().get(key))) mem = entry->mem_obj; if ((p = whichPeer(from))) diff --git a/src/net_db.cc b/src/net_db.cc index 302caf5129..74a6d86f09 100644 --- a/src/net_db.cc +++ b/src/net_db.cc @@ -1,6 +1,6 @@ /* - * $Id: net_db.cc,v 1.173 2003/09/01 03:49:39 robertc Exp $ + * $Id: net_db.cc,v 1.174 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 38 Network Measurement Database * AUTHOR: Duane Wessels @@ -445,6 +445,16 @@ sortPeerByRtt(const void *A, const void *B) return 0; } +static void +netdbPath(char *path) +{ + /* this is completely wrong. the netdb location should be memoised + * separately from the cache dirs, and also be settable in + * squid.conf RBC 20041225 + */ + snprintf(path, SQUID_MAXPATHLEN, "%s/netdb_state", Config.cacheSwap.swapDirs[0]->path); +} + static void netdbSaveState(void *foo) { @@ -455,7 +465,7 @@ netdbSaveState(void *foo) struct timeval start = current_time; int count = 0; - snprintf(path, SQUID_MAXPATHLEN, "%s/netdb_state", storeSwapDir(0)); + netdbPath(path); /* * This was nicer when we were using stdio, but thanks to * Solaris bugs, its a bad idea. fopen can fail if more than @@ -522,7 +532,7 @@ netdbReloadState(void) int count = 0; struct timeval start = current_time; - snprintf(path, SQUID_MAXPATHLEN, "%s/netdb_state", storeSwapDir(0)); + netdbPath(path); /* * This was nicer when we were using stdio, but thanks to * Solaris bugs, its a bad idea. fopen can fail if more than diff --git a/src/peer_digest.cc b/src/peer_digest.cc index fb5a0296c4..23b4e9179a 100644 --- a/src/peer_digest.cc +++ b/src/peer_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: peer_digest.cc,v 1.101 2003/09/29 10:24:01 robertc Exp $ + * $Id: peer_digest.cc,v 1.102 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 72 Peer Digest Routines * AUTHOR: Alex Rousskov @@ -355,7 +355,7 @@ peerDigestRequest(PeerDigest * pd) /* the rest is based on clientProcessExpired() */ req->flags.refresh = 1; - old_e = fetch->old_entry = storeGet(key); + old_e = fetch->old_entry = Store::Root().get(key); if (old_e) { debug(72, 5) ("peerDigestRequest: found old entry\n"); diff --git a/src/protos.h b/src/protos.h index f2fd66f7ca..568cd95b1b 100644 --- a/src/protos.h +++ b/src/protos.h @@ -1,6 +1,6 @@ /* - * $Id: protos.h,v 1.502 2004/12/21 17:52:53 robertc Exp $ + * $Id: protos.h,v 1.503 2005/01/03 16:08:26 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -55,9 +55,7 @@ SQUIDCEXTERN wordlist *wordlistDup(const wordlist *); SQUIDCEXTERN void wordlistDestroy(wordlist **); SQUIDCEXTERN void configFreeMemory(void); SQUIDCEXTERN void wordlistCat(const wordlist *, MemBuf * mb); -SQUIDCEXTERN void allocate_new_swapdir(cacheSwap *); SQUIDCEXTERN void self_destruct(void); -SQUIDCEXTERN int GetInteger(void); SQUIDCEXTERN void add_http_port(char *portspec); extern int xatoi(const char *token); extern long xatol(const char *token); @@ -701,7 +699,6 @@ extern StoreIOState::Pointer storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *) SQUIDCEXTERN void storeClose(StoreIOState::Pointer); SQUIDCEXTERN void storeRead(StoreIOState::Pointer, char *, size_t, off_t, STRCB *, void *); SQUIDCEXTERN void storeIOWrite(StoreIOState::Pointer, char const *, size_t, off_t, FREE *); -SQUIDCEXTERN void storeUnlink(StoreEntry *); /* * store_log.c @@ -739,34 +736,6 @@ SQUIDCEXTERN void storeDigestScheduleRebuild(void); SQUIDCEXTERN void storeDigestDel(const StoreEntry * entry); SQUIDCEXTERN void storeDigestReport(StoreEntry *); -/* - * store_dir.c - */ -SQUIDCEXTERN OBJH storeDirStats; -SQUIDCEXTERN char *storeDirSwapLogFile(int, const char *); -SQUIDCEXTERN char *storeSwapDir(int); -SQUIDCEXTERN char *storeSwapFullPath(int, char *); -SQUIDCEXTERN char *storeSwapSubSubDir(int, char *); -SQUIDCEXTERN const char *storeSwapPath(int); -SQUIDCEXTERN int storeDirWriteCleanLogs(int reopen); -SQUIDCEXTERN STDIRSELECT *storeDirSelectSwapDir; -SQUIDCEXTERN int storeVerifySwapDirs(void); -SQUIDCEXTERN void storeCreateSwapDirectories(void); -SQUIDCEXTERN void storeDirCloseSwapLogs(void); -SQUIDCEXTERN void storeDirCloseTmpSwapLog(int dirn); -SQUIDCEXTERN void storeDirConfigure(void); -SQUIDCEXTERN void storeDirDiskFull(sdirno); -SQUIDCEXTERN void storeDirInit(void); -SQUIDCEXTERN void storeDirOpenSwapLogs(void); -SQUIDCEXTERN void storeDirSwapLog(const StoreEntry *, int op); -SQUIDCEXTERN void storeDirUpdateSwapSize(SwapDir *, size_t size, int sign); -SQUIDCEXTERN void storeDirSync(void); -SQUIDCEXTERN void storeDirCallback(void); -SQUIDCEXTERN void storeDirLRUDelete(StoreEntry *); -SQUIDCEXTERN void storeDirLRUAdd(StoreEntry *); -SQUIDCEXTERN int storeDirGetBlkSize(const char *path, int *blksize); -SQUIDCEXTERN int storeDirGetUFSStats(const char *, int *, int *, int *, int *); - /* * store_rebuild.c */ diff --git a/src/snmp_agent.cc b/src/snmp_agent.cc index ccba201734..16ba1e31fc 100644 --- a/src/snmp_agent.cc +++ b/src/snmp_agent.cc @@ -1,6 +1,6 @@ /* - * $Id: snmp_agent.cc,v 1.89 2003/06/24 21:08:02 wessels Exp $ + * $Id: snmp_agent.cc,v 1.90 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 49 SNMP Interface * AUTHOR: Kostas Anagnostakis @@ -124,7 +124,7 @@ snmp_confFn(variable_list * Var, snint * ErrP) case CONF_ST_SWMAXSZ: Answer = snmp_var_new_integer(Var->name, Var->name_length, - (snint) Config.Swap.maxSize >> 10, + (snint) Store::Root().maxSize() >> 10, ASN_INTEGER); break; diff --git a/src/squid.h b/src/squid.h index 31e7ce5ac2..af75e9b7ef 100644 --- a/src/squid.h +++ b/src/squid.h @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.243 2004/12/21 17:28:29 robertc Exp $ + * $Id: squid.h,v 1.244 2005/01/03 16:08:26 robertc Exp $ * * AUTHOR: Duane Wessels * @@ -448,7 +448,7 @@ void free(V x) { fatal("Do not use ::free()"); } /* * I'm sick of having to keep doing this .. */ -#define INDEXSD(i) (Config.cacheSwap.swapDirs[(i)]) +#define INDEXSD(i) (Config.cacheSwap.swapDirs[(i)].getRaw()) #define FD_READ_METHOD(fd, buf, len) (*fd_table[fd].read_method)(fd, buf, len) #define FD_WRITE_METHOD(fd, buf, len) (*fd_table[fd].write_method)(fd, buf, len) diff --git a/src/stat.cc b/src/stat.cc index 9c14967dab..18b9531234 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -1,5 +1,5 @@ /* - * $Id: stat.cc,v 1.387 2004/12/27 11:04:36 serassio Exp $ + * $Id: stat.cc,v 1.388 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -49,19 +49,24 @@ * to use during static construction */ #include "comm.h" +#include "StoreSearch.h" #define DEBUG_OPENFD 1 typedef int STOBJFLT(const StoreEntry *); -typedef struct +class StatObjectsState { + +public: StoreEntry *sentry; - int bucket; STOBJFLT *filter; -} + StoreSearchPointer theSearch; + +private: + CBDATA_CLASS2(StatObjectsState); +}; -StatObjectsState; /* LOCALS */ static const char *describeStatuses(const StoreEntry *); @@ -105,7 +110,7 @@ StatCounters CountHist[N_COUNT_HIST]; static int NCountHist = 0; static StatCounters CountHourHist[N_COUNT_HOUR_HIST]; static int NCountHourHist = 0; -CBDATA_TYPE(StatObjectsState); +CBDATA_CLASS_INIT(StatObjectsState); extern unsigned int mem_pool_alloc_calls; extern unsigned int mem_pool_free_calls; @@ -335,10 +340,8 @@ statObjects(void *data) { StatObjectsState *state = static_cast(data); StoreEntry *e; - hash_link *link_ptr = NULL; - hash_link *link_next = NULL; - if (state->bucket >= store_hash_buckets) { + if (state->theSearch->isDone()) { state->sentry->complete(); storeUnlockObject(state->sentry); cbdataFree(state); @@ -352,39 +355,36 @@ statObjects(void *data) return; } - debug(49, 3) ("statObjects: Bucket #%d\n", state->bucket); - link_next = hash_get_bucket(store_table, state->bucket); - - if (link_next) { - MemBuf mb; - memBufDefInit(&mb); + storeBuffer(state->sentry); + size_t statCount = 0; + MemBuf mb; + memBufDefInit(&mb); - while (NULL != (link_ptr = link_next)) { - link_next = link_ptr->next; - e = (StoreEntry *) link_ptr; + while (statCount++ < static_cast(Config.Store.objectsPerBucket) && state-> + theSearch->next()) { + e = state->theSearch->currentItem(); - if (state->filter && 0 == state->filter(e)) - continue; + if (state->filter && 0 == state->filter(e)) + continue; - statStoreEntry(&mb, e); - } + statStoreEntry(&mb, e); + } + if (mb.size) storeAppend(state->sentry, mb.buf, mb.size); - memBufClean(&mb); - } + memBufClean(&mb); - state->bucket++; eventAdd("statObjects", statObjects, state, 0.0, 1); } static void statObjectsStart(StoreEntry * sentry, STOBJFLT * filter) { - StatObjectsState *state; - state = cbdataAlloc(StatObjectsState); + StatObjectsState *state = new StatObjectsState; state->sentry = sentry; state->filter = filter; storeLockObject(sentry); + state->theSearch = Store::Root().search(NULL, NULL); eventAdd("statObjects", statObjects, state, 0.0, 1); } @@ -965,7 +965,6 @@ statInit(void) { int i; debug(18, 5) ("statInit: Initializing...\n"); - CBDATA_INIT_TYPE(StatObjectsState); for (i = 0; i < N_COUNT_HIST; i++) statCountersInit(&CountHist[i]); diff --git a/src/store.cc b/src/store.cc index 7df3a5ebee..41282ecb00 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.577 2004/08/30 05:12:31 robertc Exp $ + * $Id: store.cc,v 1.578 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 20 Storage Manager * AUTHOR: Harvest Derived @@ -97,10 +97,7 @@ extern OBJH storeIOStats; static void storeGetMemSpace(int); static void storeHashDelete(StoreEntry *); static void destroy_MemObject(StoreEntry *); -static FREE destroyStoreEntry; static void storePurgeMem(StoreEntry *); -static void storeEntryReferenced(StoreEntry *); -static void storeEntryDereferenced(StoreEntry *); static int getKeyCounter(void); static int storeKeepInMemory(const StoreEntry *); static OBJH storeCheckCachableStats; @@ -112,6 +109,45 @@ static EVH storeLateRelease; static Stack LateReleaseStack; MemImplementingAllocator *StoreEntry::pool = NULL; +StorePointer Store::CurrentRoot = NULL; + +void +Store::Root(Store * aRoot) +{ + CurrentRoot = aRoot; +} + +void +Store::Root(StorePointer aRoot) +{ + Root(aRoot.getRaw()); +} + +void +Store::Stats(StoreEntry * output) +{ + assert (output); + Root().stat(*output); +} + +void +Store::create() +{} + +void +Store::diskFull() +{} + +void +Store::sync() +{} + +void +Store::unlink (StoreEntry &anEntry) +{ + fatal("Store::unlink on invalid Store\n"); +} + void * StoreEntry::operator new (size_t bytecount) { @@ -285,24 +321,28 @@ StoreEntry::storeClientType() const return STORE_DISK_CLIENT; } -StoreEntry * -new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) +StoreEntry::StoreEntry() { - StoreEntry *e = NULL; - e = new StoreEntry; + mem_obj = NULL; - if (mem_obj_flag) - e->mem_obj = new MemObject(url, log_url); + debugs(20, 3, "newStoreEntry: returning " << this); - debug(20, 3) ("newStoreEntry: returning %p\n", e); + expires = lastmod = lastref = timestamp = -1; - e->expires = e->lastmod = e->lastref = e->timestamp = -1; + swap_filen = -1; + swap_dirn = -1; +} - e->swap_filen = -1; +StoreEntry::StoreEntry(const char *url, const char *log_url) +{ + mem_obj = new MemObject(url, log_url); - e->swap_dirn = -1; + debugs(20, 3, "newStoreEntry: returning " << this); - return e; + expires = lastmod = lastref = timestamp = -1; + + swap_filen = -1; + swap_dirn = -1; } static void @@ -314,10 +354,10 @@ destroy_MemObject(StoreEntry * e) delete mem; } -static void +void destroyStoreEntry(void *data) { - StoreEntry *e = static_cast(data); + StoreEntry *e = static_cast(static_cast(data)); debug(20, 3) ("destroyStoreEntry: destroying %p\n", e); assert(e != NULL); @@ -371,36 +411,13 @@ storePurgeMem(StoreEntry * e) storeRelease(e); } -static void -storeEntryReferenced(StoreEntry * e) -{ - /* Notify the fs that we're referencing this object again */ - - if (e->swap_dirn > -1) - INDEXSD(e->swap_dirn)->reference(*e); - - /* Notify the memory cache that we're referencing this object again */ - if (e->mem_obj) { - if (mem_policy->Referenced) - mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl); - } -} - -static void -storeEntryDereferenced(StoreEntry * e) -{ - /* Notify the fs that we're not referencing this object any more */ - - if (e->swap_filen > -1) - INDEXSD(e->swap_dirn)->dereference(*e); - - /* Notify the memory cache that we're not referencing this object any more */ - if (e->mem_obj) { - if (mem_policy->Dereferenced) - mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl); - } -} - +/* RBC 20050104 this is wrong- memory ref counting + * is not at all equivalent to the store 'usage' concept + * which the replacement policies should be acting upon. + * specifically, object iteration within stores needs + * memory ref counting to prevent race conditions, + * but this should not influence store replacement. + */ void storeLockObject(StoreEntry * e) { @@ -408,7 +425,7 @@ storeLockObject(StoreEntry * e) debug(20, 3) ("storeLockObject: key '%s' count=%d\n", e->getMD5Text(), (int) e->lock_count); e->lastref = squid_curtime; - storeEntryReferenced(e); + Store::Root().reference(*e); } void @@ -460,11 +477,11 @@ storeUnlockObject(StoreEntry * e) if (EBIT_TEST(e->flags, RELEASE_REQUEST)) storeRelease(e); else if (storeKeepInMemory(e)) { - storeEntryDereferenced(e); + Store::Root().dereference(*e); storeSetMemStatus(e, IN_MEMORY); e->mem_obj->unlinkRequest(); } else { - storeEntryDereferenced(e); + Store::Root().dereference(*e); if (EBIT_TEST(e->flags, KEY_PRIVATE)) debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); @@ -476,18 +493,6 @@ storeUnlockObject(StoreEntry * e) return 0; } -/* Lookup an object in the cache. - * return just a reference to object, don't start swapping in yet. */ -StoreEntry * -storeGet(const cache_key * key) -{ - PROF_start(storeGet); - debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); - StoreEntry *p = static_cast(hash_lookup(store_table, key)); - PROF_stop(storeGet); - return p; -} - void StoreEntry::getPublicByRequestMethod (StoreClient *aClient, HttpRequest * request, const method_t method) { @@ -527,13 +532,13 @@ StoreEntry::getPublic (StoreClient *aClient, const char *uri, const method_t met StoreEntry * storeGetPublic(const char *uri, const method_t method) { - return storeGet(storeKeyPublic(uri, method)); + return Store::Root().get(storeKeyPublic(uri, method)); } StoreEntry * storeGetPublicByRequestMethod(HttpRequest * req, const method_t method) { - return storeGet(storeKeyPublicByRequestMethod(req, method)); + return Store::Root().get(storeKeyPublicByRequestMethod(req, method)); } StoreEntry * @@ -559,6 +564,15 @@ getKeyCounter(void) return key_counter; } +/* RBC 20050104 AFAICT this should become simpler: + * rather than reinserting with a special key it should be marked + * as 'released' and then cleaned up when refcounting indicates. + * the StoreHashIndex could well implement its 'released' in the + * current manner. + * Also, clean log writing should skip over ia,t + * Otherwise, we need a 'remove from the index but not the store + * concept'. + */ void storeSetPrivateKey(StoreEntry * e) { @@ -723,7 +737,7 @@ storeCreateEntry(const char *url, const char *log_url, request_flags flags, meth MemObject *mem = NULL; debug(20, 3) ("storeCreateEntry: '%s'\n", url); - e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url); + e = new StoreEntry(url, log_url); e->lock_count = 1; /* Note lock here w/o calling storeLock() */ mem = e->mem_obj; mem->method = method; @@ -1147,6 +1161,22 @@ storeGetMemSpace(int size) PROF_stop(storeGetMemSpace); } + +/* thunk through to Store::Root().maintain(). Note that this would be better still + * if registered against the root store itself, but that requires more complex + * update logic - bigger fish to fry first. Long term each store when + * it becomes active will self register + */ +void +Store::Maintain(void *notused) +{ + Store::Root().maintain(); + + /* Reregister a maintain event .. */ + eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1); + +} + /* The maximum objects to scan for maintain storage space */ #define MAINTAIN_MAX_SCAN 1024 #define MAINTAIN_MAX_REMOVE 64 @@ -1158,35 +1188,23 @@ storeGetMemSpace(int size) * This should get called 1/s from main(). */ void -storeMaintainSwapSpace(void *datanotused) +StoreController::maintain() { - int i; - SwapDir *SD; static time_t last_warn_time = 0; PROF_start(storeMaintainSwapSpace); - /* walk each fs */ + swapDir->maintain(); - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - /* call the maintain function .. */ - SD = INDEXSD(i); - /* XXX FixMe: This should be done "in parallell" on the different - * cache_dirs, not one at a time. - */ - SD->maintainfs(); - } + /* this should be emitted by the oversize dir, not globally */ - if (store_swap_size > Config.Swap.maxSize) { + if (store_swap_size > Store::Root().maxSize()) { if (squid_curtime - last_warn_time > 10) { - debug(20, 0) ("WARNING: Disk space over limit: %lu KB > %lu KB\n", - (long unsigned) store_swap_size, (long unsigned) Config.Swap.maxSize); + debugs(20, 0, "WARNING: Disk space over limit: " << store_swap_size << " KB > " + << Store::Root().maxSize() << " KB"); last_warn_time = squid_curtime; } } - /* Reregister a maintain event .. */ - eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); - PROF_stop(storeMaintainSwapSpace); } @@ -1225,18 +1243,18 @@ storeRelease(StoreEntry * e) PROF_stop(storeRelease); return; } else { - destroyStoreEntry(e); + destroyStoreEntry(static_cast(e)); } } storeLog(STORE_LOG_RELEASE, e); if (e->swap_filen > -1) { - storeUnlink(e); + e->unlink(); if (e->swap_status == SWAPOUT_DONE) if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, -1); + e->store()->updateSize(e->swap_file_sz, -1); if (!EBIT_TEST(e->flags, KEY_PRIVATE)) storeDirSwapLog(e, SWAP_LOG_DEL); @@ -1250,7 +1268,7 @@ storeRelease(StoreEntry * e) } storeSetMemStatus(e, NOT_IN_MEMORY); - destroyStoreEntry(e); + destroyStoreEntry(static_cast(e)); PROF_stop(storeRelease); } @@ -1357,40 +1375,19 @@ StoreEntry::validLength() const return 0; } -static void -storeInitHashValues(void) -{ - long int i; - /* Calculate size of hash table (maximum currently 64k buckets). */ - i = Config.Swap.maxSize / Config.Store.avgObjectSize; - debug(20, 1) ("Swap maxSize %ld KB, estimated %ld objects\n", - (long int) Config.Swap.maxSize, i); - i /= Config.Store.objectsPerBucket; - debug(20, 1) ("Target number of buckets: %ld\n", i); - /* ideally the full scan period should be configurable, for the - * moment it remains at approximately 24 hours. */ - store_hash_buckets = storeKeyHashBuckets(i); - debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets); - debug(20, 1) ("Max Mem size: %ld KB\n", (long int) Config.memMaxSize >> 10); - debug(20, 1) ("Max Swap size: %ld KB\n", (long int) Config.Swap.maxSize); -} - void storeInit(void) { storeKeyInit(); - storeInitHashValues(); - store_table = hash_create(storeKeyHashCmp, - store_hash_buckets, storeKeyHashHash); mem_policy = createRemovalPolicy(Config.memPolicy); storeDigestInit(); storeLogOpen(); eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); - storeDirInit(); + Store::Root().init(); storeRebuildStart(); cachemgrRegister("storedir", "Store Directory Stats", - storeDirStats, 0, 1); + Store::Stats, 0, 1); cachemgrRegister("store_check_cachable_stats", "storeCheckCachable() Stats", storeCheckCachableStats, 0, 1); @@ -1402,9 +1399,9 @@ storeInit(void) void storeConfigure(void) { - store_swap_high = (long) (((float) Config.Swap.maxSize * + store_swap_high = (long) (((float) Store::Root().maxSize() * (float) Config.Swap.highWaterMark) / (float) 100); - store_swap_low = (long) (((float) Config.Swap.maxSize * + store_swap_low = (long) (((float) Store::Root().maxSize() * (float) Config.Swap.lowWaterMark) / (float) 100); store_pages_max = Config.memMaxSize / SM_PAGE_SIZE; } @@ -1448,9 +1445,7 @@ storeNegativeCache(StoreEntry * e) void storeFreeMemory(void) { - hashFreeItems(store_table, destroyStoreEntry); - hashFreeMemory(store_table); - store_table = NULL; + Store::Root(NULL); #if USE_CACHE_DIGESTS if (store_digest) @@ -1887,6 +1882,18 @@ StoreEntry::modifiedSince(HttpRequest * request) const } } +StorePointer +StoreEntry::store() const +{ + assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured); + return INDEXSD(swap_dirn); +} + +void +StoreEntry::unlink() +{ + store()->unlink(*this); +} /* NullStoreEntry */ diff --git a/src/store_digest.cc b/src/store_digest.cc index 532cfa0fd6..50c21b5d34 100644 --- a/src/store_digest.cc +++ b/src/store_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: store_digest.cc,v 1.58 2003/09/29 10:24:02 robertc Exp $ + * $Id: store_digest.cc,v 1.59 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 71 Store Digest Manager * AUTHOR: Alex Rousskov @@ -47,23 +47,25 @@ #include "HttpRequest.h" #include "HttpReply.h" #include "MemObject.h" +#include "StoreSearch.h" /* * local types */ -typedef struct +class StoreDigestState { + +public: StoreDigestCBlock cblock; int rebuild_lock; /* bucket number */ generic_cbdata *rewrite_lock; /* points to store entry with the digest */ - int rebuild_offset; + StoreSearchPointer theSearch; int rewrite_offset; int rebuild_count; int rewrite_count; -} +}; -StoreDigestState; typedef struct { @@ -321,7 +323,7 @@ storeDigestRebuildResume(void) { assert(sd_state.rebuild_lock); assert(!sd_state.rewrite_lock); - sd_state.rebuild_offset = 0; + sd_state.theSearch = Store::Root().search(NULL, NULL); /* resize or clear */ if (!storeDigestResize()) @@ -352,28 +354,19 @@ storeDigestRebuildFinish(void) static void storeDigestRebuildStep(void *datanotused) { - int bcount = (int) ceil((double) store_hash_buckets * - (double) Config.digest.rebuild_chunk_percentage / 100.0); + /* TODO: call Store::Root().size() to determine this.. */ + int count = Config.Store.objectsPerBucket * (int) ceil((double) store_hash_buckets * + (double) Config.digest.rebuild_chunk_percentage / 100.0); assert(sd_state.rebuild_lock); - if (sd_state.rebuild_offset + bcount > store_hash_buckets) - bcount = store_hash_buckets - sd_state.rebuild_offset; - - debug(71, 3) ("storeDigestRebuildStep: buckets: %d offset: %d chunk: %d buckets\n", - store_hash_buckets, sd_state.rebuild_offset, bcount); + debug(71, 3) ("storeDigestRebuildStep: buckets: %d entries to check: %d\n", + store_hash_buckets, count); - while (bcount--) { - hash_link *link_ptr = hash_get_bucket(store_table, sd_state.rebuild_offset); - - for (; link_ptr; link_ptr = link_ptr->next) { - storeDigestAdd((StoreEntry *) link_ptr); - } - - sd_state.rebuild_offset++; - } + while (count-- && !sd_state.theSearch->isDone() && sd_state.theSearch->next()) + storeDigestAdd(sd_state.theSearch->currentItem()); /* are we done ? */ - if (sd_state.rebuild_offset >= store_hash_buckets) + if (sd_state.theSearch->isDone()) storeDigestRebuildFinish(); else eventAdd("storeDigestRebuildStep", storeDigestRebuildStep, NULL, 0.0, 1); @@ -520,7 +513,7 @@ storeDigestCalcCap(void) * the bits are off). However, we do not have a formula to calculate the * number of _entries_ we want to pre-allocate for. */ - const int hi_cap = Config.Swap.maxSize / Config.Store.avgObjectSize; + const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize; const int lo_cap = 1 + store_swap_size / Config.Store.avgObjectSize; const int e_count = StoreEntry::inUseCount(); int cap = e_count ? e_count : hi_cap; diff --git a/src/store_dir.cc b/src/store_dir.cc index 2676849a14..5a572f1e6d 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir.cc,v 1.149 2003/07/29 12:20:59 robertc Exp $ + * $Id: store_dir.cc,v 1.150 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -55,10 +55,16 @@ #include #endif -static int storeDirValidSwapDirSize(int, ssize_t); +#include "StoreHashIndex.h" + static STDIRSELECT storeDirSelectSwapDirRoundRobin; static STDIRSELECT storeDirSelectSwapDirLeastLoad; -static void startOneStoreCreation(SwapDir &swapDir); + +StoreController::StoreController() : swapDir (new StoreHashIndex()) +{} + +StoreController::~StoreController() +{} /* * This function pointer is set according to 'store_dir_select_algorithm' @@ -67,12 +73,9 @@ static void startOneStoreCreation(SwapDir &swapDir); STDIRSELECT *storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad; void -storeDirInit(void) +StoreController::init() { - int i; - - for (i = 0; i < Config.cacheSwap.n_configured; i++) - INDEXSD(i)->init(); + swapDir->init(); if (0 == strcasecmp(Config.store_dir_select_algorithm, "round-robin")) { storeDirSelectSwapDir = storeDirSelectSwapDirRoundRobin; @@ -84,7 +87,7 @@ storeDirInit(void) } void -startOneStoreCreation(SwapDir &swapDir) +StoreController::createOneStore(Store &aStore) { /* * On Windows, fork() is not available. @@ -98,7 +101,7 @@ startOneStoreCreation(SwapDir &swapDir) #endif - swapDir.newFileSystem(); + aStore.create(); #ifndef _SQUID_MSWIN_ @@ -108,10 +111,9 @@ startOneStoreCreation(SwapDir &swapDir) } void -storeCreateSwapDirectories(void) +StoreController::create() { - for (int i = 0; i < Config.cacheSwap.n_configured; i++) - startOneStoreCreation(*INDEXSD(i)); + swapDir->create(); #ifndef _SQUID_MSWIN_ @@ -140,31 +142,27 @@ storeCreateSwapDirectories(void) * will return true here are ones that have max_obj_size = -1, * ie any-sized-object swapdirs. This is a good thing. */ -static int -storeDirValidSwapDirSize(int swapdir, ssize_t objsize) +bool +SwapDir::objectSizeIsAcceptable(ssize_t objsize) const { /* * If the swapdir's max_obj_size is -1, then it definitely can */ - if (INDEXSD(swapdir)->max_objsize == -1) - return 1; + if (max_objsize == -1) + return true; /* * If the object size is -1, then if the storedir isn't -1 we * can't store it */ - if ((objsize == -1) && - (INDEXSD(swapdir)->max_objsize != -1)) - return 0; + if ((objsize == -1) && (max_objsize != -1)) + return false; /* * Else, make sure that the max object size is larger than objsize */ - if (INDEXSD(swapdir)->max_objsize > objsize) - return 1; - else - return 0; + return max_objsize > objsize; } @@ -179,14 +177,14 @@ storeDirSelectSwapDirRoundRobin(const StoreEntry * e) static int dirn = 0; int i; int load; - SwapDir *sd; + RefCount sd; ssize_t objsize = (ssize_t) objectLen(e); for (i = 0; i <= Config.cacheSwap.n_configured; i++) { if (++dirn >= Config.cacheSwap.n_configured) dirn = 0; - sd = INDEXSD(dirn); + sd = dynamic_cast(INDEXSD(dirn)); if (sd->flags.read_only) continue; @@ -194,7 +192,7 @@ storeDirSelectSwapDirRoundRobin(const StoreEntry * e) if (sd->cur_size > sd->max_size) continue; - if (!storeDirValidSwapDirSize(dirn, objsize)) + if (!sd->objectSizeIsAcceptable(objsize)) continue; /* check for error or overload condition */ @@ -233,7 +231,7 @@ storeDirSelectSwapDirLeastLoad(const StoreEntry * e) int load; int dirn = -1; int i; - SwapDir *SD; + RefCount SD; /* Calculate the object size */ objsize = (ssize_t) objectLen(e); @@ -242,7 +240,7 @@ storeDirSelectSwapDirLeastLoad(const StoreEntry * e) objsize += e->mem_obj->swap_hdr_sz; for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = INDEXSD(i); + SD = dynamic_cast(INDEXSD(i)); SD->flags.selected = 0; load = SD->canStore(*e); @@ -250,7 +248,7 @@ storeDirSelectSwapDirLeastLoad(const StoreEntry * e) continue; } - if (!storeDirValidSwapDirSize(i, objsize)) + if (!SD->objectSizeIsAcceptable(objsize)) continue; if (SD->flags.read_only) @@ -284,20 +282,11 @@ storeDirSelectSwapDirLeastLoad(const StoreEntry * e) } if (dirn >= 0) - INDEXSD(dirn)->flags.selected = 1; + dynamic_cast(INDEXSD(dirn))->flags.selected = 1; return dirn; } - - -char * -storeSwapDir(int dirn) -{ - assert(0 <= dirn && dirn < Config.cacheSwap.n_configured); - return INDEXSD(dirn)->path; -} - /* * An entry written to the swap log MUST have the following * properties. @@ -328,15 +317,21 @@ storeDirSwapLog(const StoreEntry * e, int op) e->swap_dirn, e->swap_filen); - INDEXSD(e->swap_dirn)->logEntry(*e, op); + dynamic_cast(INDEXSD(e->swap_dirn))->logEntry(*e, op); +} + +void +StoreController::updateSize(size_t size, int sign) +{ + fatal("StoreController has no independent size\n"); } void -storeDirUpdateSwapSize(SwapDir * SD, size_t size, int sign) +SwapDir::updateSize(size_t size, int sign) { - int blks = (size + SD->fs.blksize - 1) / SD->fs.blksize; - int k = (blks * SD->fs.blksize >> 10) * sign; - SD->cur_size += k; + int blks = (size + fs.blksize - 1) / fs.blksize; + int k = (blks * fs.blksize >> 10) * sign; + cur_size += k; store_swap_size += k; if (sign > 0) @@ -346,86 +341,62 @@ storeDirUpdateSwapSize(SwapDir * SD, size_t size, int sign) } void -storeDirStats(StoreEntry * sentry) +StoreController::stat(StoreEntry &output) const { - int i; - SwapDir *SD; - assert (sentry); - - storeAppendPrintf(sentry, "Store Directory Statistics:\n"); - storeAppendPrintf(sentry, "Store Entries : %lu\n", + storeAppendPrintf(&output, "Store Directory Statistics:\n"); + storeAppendPrintf(&output, "Store Entries : %lu\n", (unsigned long int)StoreEntry::inUseCount()); - storeAppendPrintf(sentry, "Maximum Swap Size : %8ld KB\n", - (long int) Config.Swap.maxSize); - storeAppendPrintf(sentry, "Current Store Swap Size: %8lu KB\n", + storeAppendPrintf(&output, "Maximum Swap Size : %8ld KB\n", + (long int) maxSize()); + storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n", store_swap_size); - storeAppendPrintf(sentry, "Current Capacity : %d%% used, %d%% free\n", - percent((int) store_swap_size, (int) Config.Swap.maxSize), - percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize)); + storeAppendPrintf(&output, "Current Capacity : %d%% used, %d%% free\n", + percent((int) store_swap_size, (int) maxSize()), + percent((int) (maxSize() - store_swap_size), (int) maxSize())); /* FIXME Here we should output memory statistics */ - /* Now go through each swapdir, calling its statfs routine */ - - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - storeAppendPrintf(sentry, "\n"); - SD = INDEXSD(i); - storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type(), - storeSwapDir(i)); - storeAppendPrintf(sentry, "FS Block Size %d Bytes\n", - SD->fs.blksize); - SD->statfs(*sentry); - - if (SD->repl) { - storeAppendPrintf(sentry, "Removal policy: %s\n", SD->repl->_type); - - if (SD->repl->Stats) - SD->repl->Stats(SD->repl, sentry); - } - } + /* now the swapDir */ + swapDir->stat(output); } -void -storeDirConfigure(void) +/* if needed, this could be taught to cache the result */ +size_t +StoreController::maxSize() const { - SwapDir *SD; - int i; - Config.Swap.maxSize = 0; + /* TODO: include memory cache ? */ + return swapDir->maxSize(); +} - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - SD = INDEXSD(i); - Config.Swap.maxSize += SD->max_size; - SD->low_size = (int) (((float) SD->max_size * - (float) Config.Swap.lowWaterMark) / 100.0); - } +size_t +StoreController::minSize() const +{ + /* TODO: include memory cache ? */ + return swapDir->minSize(); } void -storeDirDiskFull(sdirno dirn) +SwapDir::diskFull() { - SwapDir *SD = INDEXSD(dirn); - assert(0 <= dirn && dirn < Config.cacheSwap.n_configured); - - if (SD->cur_size >= SD->max_size) + if (cur_size >= max_size) return; - SD->max_size = SD->cur_size; + max_size = cur_size; - debug(20, 1) ("WARNING: Shrinking cache_dir #%d to %d KB\n", - dirn, SD->cur_size); + debugs(20, 1, "WARNING: Shrinking cache_dir #" << index << " to " << cur_size << " KB"); } void storeDirOpenSwapLogs(void) { for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) - INDEXSD(dirn)->openLog(); + dynamic_cast(INDEXSD(dirn))->openLog(); } void storeDirCloseSwapLogs(void) { for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) - INDEXSD(dirn)->closeLog(); + dynamic_cast(INDEXSD(dirn))->closeLog(); } /* @@ -445,7 +416,7 @@ storeDirWriteCleanLogs(int reopen) struct timeval start; double dt; - SwapDir *sd; + RefCount sd; int dirn; int notdone = 1; @@ -460,7 +431,7 @@ storeDirWriteCleanLogs(int reopen) start = current_time; for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { - sd = INDEXSD(dirn); + sd = dynamic_cast(INDEXSD(dirn)); if (sd->writeCleanStart() < 0) { debug(20, 1) ("log.clean.start() failed for dir #%d\n", sd->index); @@ -475,7 +446,7 @@ storeDirWriteCleanLogs(int reopen) notdone = 0; for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { - sd = INDEXSD(dirn); + sd = dynamic_cast(INDEXSD(dirn)); if (NULL == sd->cleanLog) continue; @@ -501,7 +472,7 @@ storeDirWriteCleanLogs(int reopen) /* Flush */ for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) - INDEXSD(dirn)->writeCleanDone(); + dynamic_cast(INDEXSD(dirn))->writeCleanDone(); if (reopen) storeDirOpenSwapLogs(); @@ -518,49 +489,41 @@ storeDirWriteCleanLogs(int reopen) return n; } -/* - * sync all avaliable fs'es .. - */ +StoreSearch * +StoreController::search(String const url, HttpRequest *request) +{ + /* cheat, for now you can't search the memory hot cache */ + return swapDir->search(url, request); +} + +StorePointer +StoreHashIndex::store(int const x) const +{ + return INDEXSD(x); +} + void -storeDirSync(void) +StoreController::sync(void) { - for (int i = 0; i < Config.cacheSwap.n_configured; ++i) - INDEXSD(i)->sync(); + /* sync mem cache? */ + swapDir->sync(); } /* * handle callbacks all avaliable fs'es */ -void -storeDirCallback(void) +int +StoreController::callback() { - int j; - static int ndir = 0; - /* This will likely double count. Thats ok. */ PROF_start(storeDirCallback); - do { - j = 0; - - for (int i = 0; i < Config.cacheSwap.n_configured; i++) { - if (ndir >= Config.cacheSwap.n_configured) - ndir = ndir % Config.cacheSwap.n_configured; - - SwapDir *SD = INDEXSD(ndir); - - ++ndir; - - j += SD->callback(); - - if (j > 100) - fatal ("too much io\n"); - } - } while (j > 0); - - ndir++; + /* mem cache callbacks ? */ + int result = swapDir->callback(); PROF_stop(storeDirCallback); + + return result; } int @@ -635,3 +598,355 @@ storeDirGetUFSStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, return 0; } + +void +allocate_new_swapdir(_SquidConfig::_cacheSwap * swap) +{ + if (swap->swapDirs == NULL) { + swap->n_allocated = 4; + swap->swapDirs = static_cast(xcalloc(swap->n_allocated, sizeof(StorePointer))); + } + + if (swap->n_allocated == swap->n_configured) { + StorePointer *tmp; + swap->n_allocated <<= 1; + tmp = static_cast(xcalloc(swap->n_allocated, sizeof(StorePointer))); + xmemcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *)); + xfree(swap->swapDirs); + swap->swapDirs = tmp; + } +} + +void +free_cachedir(_SquidConfig::_cacheSwap * swap) +{ + int i; + /* DON'T FREE THESE FOR RECONFIGURE */ + + if (reconfiguring) + return; + + for (i = 0; i < swap->n_configured; i++) { + /* TODO XXX this lets the swapdir free resources asynchronously + * swap->swapDirs[i]->deactivate(); + * but there may be such a means already. + * RBC 20041225 + */ + swap->swapDirs[i] = NULL; + } + + safe_free(swap->swapDirs); + swap->swapDirs = NULL; + swap->n_allocated = 0; + swap->n_configured = 0; +} + +/* this should be a virtual method on StoreEntry, + * i.e. e->referenced() + * so that the entry can notify the creating Store + */ +void +StoreController::reference(StoreEntry &e) +{ + /* Notify the fs that we're referencing this object again */ + + if (e.swap_dirn > -1) + e.store()->reference(e); + + /* Notify the memory cache that we're referencing this object again */ + if (e.mem_obj) { + if (mem_policy->Referenced) + mem_policy->Referenced(mem_policy, &e, &e.mem_obj->repl); + } +} + +void +StoreController::dereference(StoreEntry & e) +{ + /* Notify the fs that we're not referencing this object any more */ + + if (e.swap_filen > -1) + e.store()->dereference(e); + + /* Notify the memory cache that we're not referencing this object any more */ + if (e.mem_obj) { + if (mem_policy->Dereferenced) + mem_policy->Dereferenced(mem_policy, &e, &e.mem_obj->repl); + } +} + +StoreEntry * + +StoreController::get + (const cache_key *key) +{ + + return swapDir->get + (key); +} + +void + +StoreController::get + (String const key, STOREGETCLIENT callback, void *cbdata) +{ + fatal("not implemented"); +} + +StoreHashIndex::StoreHashIndex() +{ + assert (store_table == NULL); +} + +StoreHashIndex::~StoreHashIndex() +{ + if (store_table) { + hashFreeItems(store_table, destroyStoreEntry); + hashFreeMemory(store_table); + store_table = NULL; + } +} + +int +StoreHashIndex::callback() +{ + int result = 0; + int j; + static int ndir = 0; + + do { + j = 0; + + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + if (ndir >= Config.cacheSwap.n_configured) + ndir = ndir % Config.cacheSwap.n_configured; + + int temp_result = store(ndir)->callback(); + + ++ndir; + + j += temp_result; + + result += temp_result; + + if (j > 100) + fatal ("too much io\n"); + } + } while (j > 0); + + ndir++; + + return result; +} + +void +StoreHashIndex::create() +{ + for (int i = 0; i < Config.cacheSwap.n_configured; i++) + store(i)->create(); +} + +/* Lookup an object in the cache. + * return just a reference to object, don't start swapping in yet. */ +StoreEntry * + +StoreHashIndex::get + (const cache_key *key) +{ + PROF_start(storeGet); + debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); + StoreEntry *p = static_cast(hash_lookup(store_table, key)); + PROF_stop(storeGet); + return p; +} + +void + +StoreHashIndex::get + (String const key, STOREGETCLIENT callback, void *cbdata) +{ + fatal("not implemented"); +} + +void +StoreHashIndex::init() +{ + /* Calculate size of hash table (maximum currently 64k buckets). */ + /* this is very bogus, its specific to the any Store maintaining an + * in-core index, not global */ + size_t buckets = Store::Root().maxSize() / Config.Store.avgObjectSize; + debugs(20, 1, "Swap maxSize " << Store::Root().maxSize() << + " KB, estimated " << buckets << " objects\n"); + buckets /= Config.Store.objectsPerBucket; + debugs(20, 1, "Target number of buckets: " << buckets); + /* ideally the full scan period should be configurable, for the + * moment it remains at approximately 24 hours. */ + store_hash_buckets = storeKeyHashBuckets(buckets); + debugs(20, 1, "Using " << store_hash_buckets << " Store buckets"); + debugs(20, 1, "Max Mem size: " << ( Config.memMaxSize >> 10) << " KB"); + debugs(20, 1, "Max Swap size: " << Store::Root().maxSize() << " KB"); + + store_table = hash_create(storeKeyHashCmp, + store_hash_buckets, storeKeyHashHash); + + for (int i = 0; i < Config.cacheSwap.n_configured; i++) + /* this starts a search of the store dirs, loading their + * index. under the new Store api this should be + * driven by the StoreHashIndex, not by each store. + * Step 1: make the store rebuilds use a search internally + */ + store(i)->init(); + +} + +size_t +StoreHashIndex::maxSize() const +{ + int i; + size_t result = 0; + + for (i = 0; i < Config.cacheSwap.n_configured; i++) + result += store(i)->maxSize(); + + return result; +} + +size_t +StoreHashIndex::minSize() const +{ + size_t result = 0; + + for (int i = 0; i < Config.cacheSwap.n_configured; i++) + result += store(i)->minSize(); + + return result; +} + +void +StoreHashIndex::stat(StoreEntry & output) const +{ + int i; + + /* Now go through each store, calling its stat routine */ + + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + storeAppendPrintf(&output, "\n"); + store(i)->stat(output); + } +} + +void +StoreHashIndex::reference(StoreEntry&) +{} + +void +StoreHashIndex::dereference(StoreEntry&) +{} + +void +StoreHashIndex::maintain() +{ + int i; + /* walk each fs */ + + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + /* XXX FixMe: This should be done "in parallell" on the different + * cache_dirs, not one at a time. + */ + /* call the maintain function .. */ + store(i)->maintain(); + } +} + +void +StoreHashIndex::updateSize(unsigned int, int) +{} + +void +StoreHashIndex::sync() +{ + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) + store(i)->sync(); +} + +StoreSearch * +StoreHashIndex::search(String const url, HttpRequest *) +{ + if (url.size()) + fatal ("Cannot search by url yet\n"); + + return new StoreSearchHashIndex (this); +} + +CBDATA_CLASS_INIT(StoreSearchHashIndex); +StoreSearchHashIndex::StoreSearchHashIndex(RefCount aSwapDir) : sd(aSwapDir), _done (false), bucket (0) +{} + +/* do not link +StoreSearchHashIndex::StoreSearchHashIndex(StoreSearchHashIndex const &); +*/ + +StoreSearchHashIndex::~StoreSearchHashIndex() +{} + +void +StoreSearchHashIndex::next(void (callback)(void *cbdata), void *cbdata) +{ + next(); + callback (cbdata); +} + +bool +StoreSearchHashIndex::next() +{ + if (entries.size()) + entries.pop_back(); + + while (!isDone() && !entries.size()) + copyBucket(); + + return currentItem() != NULL; +} + +bool +StoreSearchHashIndex::error() const +{ + return false; +} + +bool +StoreSearchHashIndex::isDone() const +{ + return bucket >= store_hash_buckets || _done; +} + +StoreEntry * +StoreSearchHashIndex::currentItem() +{ + if (!entries.size()) + return NULL; + + return entries.back(); +} + +void +StoreSearchHashIndex::copyBucket() +{ + /* probably need to lock the store entries... + * we copy them all to prevent races on the links. */ + debugs(47, 3, "StoreSearchHashIndex::copyBucket #" << bucket); + assert (!entries.size()); + hash_link *link_ptr = NULL; + hash_link *link_next = NULL; + link_next = hash_get_bucket(store_table, bucket); + + while (NULL != (link_ptr = link_next)) { + link_next = link_ptr->next; + StoreEntry *e = (StoreEntry *) link_ptr; + + entries.push_back(e); + } + + bucket++; + debugs(47,3, "got entries: " << entries.size()); +} diff --git a/src/store_io.cc b/src/store_io.cc index 43b1c60226..4d0479ca52 100644 --- a/src/store_io.cc +++ b/src/store_io.cc @@ -33,7 +33,7 @@ storeCreate(StoreEntry * e, STIOCB * file_callback, STIOCB * close_callback, voi assert (e); ssize_t objsize; sdirno dirn; - SwapDir *SD; + RefCount SD; store_io_stats.create.calls++; /* This is just done for logging purposes */ @@ -55,7 +55,7 @@ storeCreate(StoreEntry * e, STIOCB * file_callback, STIOCB * close_callback, voi } debug(20, 2) ("storeCreate: Selected dir '%d' for obj size '%ld'\n", dirn, (long int) objsize); - SD = INDEXSD(dirn); + SD = dynamic_cast(INDEXSD(dirn)); /* Now that we have a fs to use, call its storeCreate function */ StoreIOState::Pointer sio = SD->createStoreIO(*e, file_callback, close_callback, callback_data); @@ -75,7 +75,7 @@ StoreIOState::Pointer storeOpen(StoreEntry * e, STFNCB * file_callback, STIOCB * callback, void *callback_data) { - return INDEXSD(e->swap_dirn)->openStoreIO(*e, file_callback, callback, callback_data); + return dynamic_cast(e->store().getRaw())->openStoreIO(*e, file_callback, callback, callback_data); } void @@ -101,13 +101,6 @@ storeIOWrite(StoreIOState::Pointer sio, char const *buf, size_t size, off_t offs sio->write(buf,size,offset,free_func); } -void -storeUnlink(StoreEntry * e) -{ - assert (e); - INDEXSD(e->swap_dirn)->unlink(*e); -} - /* * Make this non-static so we can register * it from storeInit(); diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc index 9384ba30a3..6e999d1112 100644 --- a/src/store_rebuild.cc +++ b/src/store_rebuild.cc @@ -1,6 +1,6 @@ /* - * $Id: store_rebuild.cc,v 1.81 2003/02/21 22:50:12 robertc Exp $ + * $Id: store_rebuild.cc,v 1.82 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 20 Store Rebuild Routines * AUTHOR: Duane Wessels @@ -36,6 +36,7 @@ #include "squid.h" #include "Store.h" #include "SwapDir.h" +#include "StoreSearch.h" static struct _store_rebuild_data counts; @@ -57,73 +58,71 @@ static store_rebuild_progress *RebuildProgress = NULL; static int storeCleanupDoubleCheck(StoreEntry * e) { - SwapDir *SD = INDEXSD(e->swap_dirn); + SwapDir *SD = dynamic_cast(INDEXSD(e->swap_dirn)); return (SD->doubleCheck(*e)); } static void storeCleanup(void *datanotused) { - static int bucketnum = -1; - static int validnum = 0; static int store_errors = 0; - int validnum_start; - StoreEntry *e; - hash_link *link_ptr = NULL; - hash_link *link_next = NULL; - validnum_start = validnum; - - while (validnum - validnum_start < 500) { - if (++bucketnum >= store_hash_buckets) { - debug(20, 1) (" Completed Validation Procedure\n"); - debug(20, 1) (" Validated %d Entries\n", validnum); - debug(20, 1) (" store_swap_size = %luk\n", store_swap_size); - store_dirs_rebuilding--; - assert(0 == store_dirs_rebuilding); - - if (opt_store_doublecheck) - assert(store_errors == 0); - - if (store_digest) - storeDigestNoteStoreReady(); - - return; - } - - link_next = hash_get_bucket(store_table, bucketnum); - - while (NULL != (link_ptr = link_next)) { - link_next = link_ptr->next; - e = (StoreEntry *) link_ptr; - - if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) - continue; - - /* - * Calling storeRelease() has no effect because we're - * still in 'store_rebuilding' state - */ - if (e->swap_filen < 0) - continue; - - if (opt_store_doublecheck) - if (storeCleanupDoubleCheck(e)) - store_errors++; - - EBIT_SET(e->flags, ENTRY_VALIDATED); - - /* - * Only set the file bit if we know its a valid entry - * otherwise, set it in the validation procedure - */ - storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, 1); - - if ((++validnum & 0x3FFFF) == 0) - debug(20, 1) (" %7d Entries Validated so far.\n", validnum); - } + static StoreSearchPointer currentSearch; + static int validated = 0; + + if (currentSearch == NULL || currentSearch->isDone()) + currentSearch = Store::Root().search(NULL, NULL); + + size_t statCount = 500; + + while (statCount-- && !currentSearch->isDone() && currentSearch->next()) { + ++validated; + StoreEntry *e; + + e = currentSearch->currentItem(); + + if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) + continue; + + /* + * Calling storeRelease() has no effect because we're + * still in 'store_rebuilding' state + */ + if (e->swap_filen < 0) + continue; + + if (opt_store_doublecheck) + if (storeCleanupDoubleCheck(e)) + store_errors++; + + EBIT_SET(e->flags, ENTRY_VALIDATED); + + /* + * Only set the file bit if we know its a valid entry + * otherwise, set it in the validation procedure + */ + e->store()->updateSize(e->swap_file_sz, 1); + + if ((++validated & 0x3FFFF) == 0) + /* TODO format the int with with a stream operator */ + debugs(20, 1, " " << validated << " Entries Validated so far."); } - eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1); + if (currentSearch->isDone()) { + debugs(20, 1, " Completed Validation Procedure"); + debugs(20, 1, " Validated " << validated << " Entries"); + debugs(20, 1, " store_swap_size = " << store_swap_size); + store_dirs_rebuilding--; + assert(0 == store_dirs_rebuilding); + + if (opt_store_doublecheck) + assert(store_errors == 0); + + if (store_digest) + storeDigestNoteStoreReady(); + + currentSearch = NULL; + } else + eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1); } /* meta data recreated from disk image in swap directory */ diff --git a/src/store_swapout.cc b/src/store_swapout.cc index 10ec9be9bf..fc8811d11b 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -1,6 +1,6 @@ /* - * $Id: store_swapout.cc,v 1.98 2004/02/12 02:31:55 adrian Exp $ + * $Id: store_swapout.cc,v 1.99 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 20 Storage Manager Swapout Functions * AUTHOR: Duane Wessels @@ -322,13 +322,15 @@ storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio) e->swap_dirn, e->swap_filen, errflag, xstrerror()); if (errflag == DISK_NO_SPACE_LEFT) { - storeDirDiskFull(e->swap_dirn); - storeDirConfigure(); + /* FIXME: this should be handle by the link from store IO to + * Store, rather than being a top level API call. + */ + e->store()->diskFull(); storeConfigure(); } if (e->swap_filen > 0) - storeUnlink(e); + e->unlink(); e->swap_filen = -1; @@ -343,7 +345,7 @@ storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio) storeUrl(e), e->swap_dirn, e->swap_filen); e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz; e->swap_status = SWAPOUT_DONE; - storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, 1); + e->store()->updateSize(e->swap_file_sz, 1); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); diff --git a/src/structs.h b/src/structs.h index 765bd464cb..634ca8850d 100644 --- a/src/structs.h +++ b/src/structs.h @@ -1,6 +1,6 @@ /* - * $Id: structs.h,v 1.503 2004/12/23 22:17:22 hno Exp $ + * $Id: structs.h,v 1.504 2005/01/03 16:08:26 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -35,6 +35,7 @@ #define SQUID_STRUCTS_H #include "config.h" +#include "RefCount.h" class dlink_node { @@ -206,20 +207,26 @@ struct _https_port_list #include "DelayConfig.h" #endif -struct _RemovalPolicySettings +class RemovalPolicySettings { + +public: char *type; wordlist *args; }; class external_acl; +class Store; + struct _SquidConfig { struct { - size_t maxSize; + /* These should be for the Store::Root instance. + * this needs pluggable parsing to be done smoothly. + */ int highWaterMark; int lowWaterMark; } @@ -614,7 +621,7 @@ struct _SquidConfig struct _cacheSwap { - SwapDir **swapDirs; + RefCount *swapDirs; int n_allocated; int n_configured; } diff --git a/src/tests/TestSwapDir.cc b/src/tests/TestSwapDir.cc new file mode 100644 index 0000000000..fd0bddcde1 --- /dev/null +++ b/src/tests/TestSwapDir.cc @@ -0,0 +1,52 @@ +#include "config.h" + +#include "squid.h" +#include "TestSwapDir.h" + +size_t +TestSwapDir::maxSize() const +{ + return 3; +} + +void +TestSwapDir::stat(StoreEntry &) const +{ + const_cast(this)->statsCalled = true; +} + +void +TestSwapDir::reconfigure(int, char*) +{} + +void +TestSwapDir::init() +{} + +int +TestSwapDir::canStore(const StoreEntry&) const +{ + return true; +} + +RefCount +TestSwapDir::createStoreIO(StoreEntry&, void (*)(void*, int, storeIOState*), void (*)(void*, int, storeIOState*), void*) +{ + return NULL; +} + +RefCount +TestSwapDir::openStoreIO(StoreEntry&, void (*)(void*, int, storeIOState*), void (*)(void*, int, storeIOState*), void*) +{ + return NULL; +} + +void +TestSwapDir::parse(int, char*) +{} + +StoreSearch * +TestSwapDir::search(String, HttpRequest *) +{ + return NULL; +} diff --git a/src/tests/TestSwapDir.h b/src/tests/TestSwapDir.h new file mode 100644 index 0000000000..db28d95b95 --- /dev/null +++ b/src/tests/TestSwapDir.h @@ -0,0 +1,31 @@ +#ifndef TEST_TESTSWAPDIR +#define TEST_TESTSWAPDIR + +#include "squid.h" +#include "SwapDir.h" + +class TestSwapDir : public SwapDir +{ + +public: + TestSwapDir() : SwapDir("test"), statsCalled (false) {} + + bool statsCalled; + + virtual size_t maxSize() const; + virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */ + + virtual void reconfigure(int, char*); + virtual void init(); + virtual int canStore(const StoreEntry&) const; + virtual RefCount createStoreIO(StoreEntry&, void + (*)(void*, int, storeIOState*), void (*)(void*, int, storeIOState*), void*); + virtual RefCount openStoreIO(StoreEntry&, void + (*)(void*, int, storeIOState*), void (*)(void*, int, storeIOState*), void*); + virtual void parse(int, char*); + virtual StoreSearch *search(String, HttpRequest *); +}; + +typedef RefCount TestSwapDirPointer; + +#endif /* TEST_TESTSWAPDIR */ diff --git a/src/tests/stub_HttpReply.cc b/src/tests/stub_HttpReply.cc new file mode 100644 index 0000000000..097e8c76d3 --- /dev/null +++ b/src/tests/stub_HttpReply.cc @@ -0,0 +1,67 @@ +/* + * $Id: stub_HttpReply.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "HttpReply.h" + +void +httpReplyAbsorb(HttpReply * rep, HttpReply * new_rep) +{ + fatal ("Not implemented"); +} + +void +httpReplySetHeaders(HttpReply * rep, HttpVersion ver, http_status status, + const char *reason, const char *ctype, int clen, time_t lmt, time_t expires) +{ + fatal ("Not implemented"); +} + +void +httpReplyPackHeadersInto(const HttpReply * rep, Packer * p) +{ + fatal ("Not implemented"); +} + +void +httpReplyReset(HttpReply * rep) +{ + fatal ("Not implemented"); +} + +void +httpBodyPackInto(const HttpBody * body, Packer * p) +{ + fatal ("Not implemented"); +} diff --git a/src/tests/stub_HttpRequest.cc b/src/tests/stub_HttpRequest.cc index cac571063f..c00dfa091c 100644 --- a/src/tests/stub_HttpRequest.cc +++ b/src/tests/stub_HttpRequest.cc @@ -1,5 +1,5 @@ /* - * $Id: stub_HttpRequest.cc,v 1.1 2004/08/30 03:29:02 robertc Exp $ + * $Id: stub_HttpRequest.cc,v 1.2 2005/01/03 16:08:27 robertc Exp $ * * DEBUG: section 28 Access Control * AUTHOR: Robert Collins @@ -51,6 +51,9 @@ requestLink(HttpRequest * request) void requestUnlink(HttpRequest * request) { + if (!request) + return; + fatal("Not implemented"); } diff --git a/src/tests/stub_MemObject.cc b/src/tests/stub_MemObject.cc new file mode 100644 index 0000000000..54d1f773c0 --- /dev/null +++ b/src/tests/stub_MemObject.cc @@ -0,0 +1,133 @@ +/* + * $Id: stub_MemObject.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "MemObject.h" +#include "HttpReply.h" + +off_t +MemObject::endOffset () const +{ + return data_hdr.endOffset(); +} + +void +MemObject::trimSwappable() +{ + fatal ("Not implemented"); +} + +void +MemObject::trimUnSwappable() +{ + fatal ("Not implemented"); +} + +off_t +MemObject::policyLowestOffsetToKeep() const +{ + fatal ("Not implemented"); + return -1; +} + +MemObject::MemObject (char const *, char const *) +{} + +HttpReply const * +MemObject::getReply() const +{ + return NULL; +} + +void +MemObject::reset() +{ + fatal ("Not implemented"); +} + +void +MemObject::delayRead(DeferredRead const &aRead) +{ + fatal ("Not implemented"); +} + +bool +MemObject::readAheadPolicyCanRead() const +{ + fatal ("Not implemented"); + return false; +} + +void +MemObject::setNoDelay(bool const newValue) +{ + fatal ("Not implemented"); +} + +MemObject::~MemObject() +{ + fatal ("Not implemented"); +} + +int +MemObject::mostBytesWanted(int max) const +{ + fatal ("Not implemented"); + return -1; +} + +void +MemObject::unlinkRequest() +{ + fatal ("Not implemented"); +} + +void +MemObject::write(StoreIOBuffer writeBuffer, STMCB *callback, void *callbackData) +{ + PROF_start(MemObject_write); + debug(19, 6) ("memWrite: offset %lu len %ld\n", (unsigned long)writeBuffer.offset, (long)writeBuffer.length); + + /* the offset is into the content, not the headers */ + writeBuffer.offset += (_reply ? _reply->hdr_sz : 0); + + /* We don't separate out mime headers yet, so ensure that the first + * write is at offset 0 - where they start + */ + assert (data_hdr.endOffset() || writeBuffer.offset == 0); + + assert (data_hdr.write (writeBuffer)); + callback (callbackData, writeBuffer); + PROF_stop(MemObject_write); +} diff --git a/src/tests/stub_comm.cc b/src/tests/stub_comm.cc new file mode 100644 index 0000000000..a92b7267e7 --- /dev/null +++ b/src/tests/stub_comm.cc @@ -0,0 +1,132 @@ +/* + * $Id: stub_comm.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "comm.h" +#include "fde.h" + +DeferredReadManager::~DeferredReadManager() +{ + /* no networked tests yet */ +} + +DeferredRead::DeferredRead (DeferrableRead *, void *, CommRead const &) +{ + fatal ("Not implemented"); +} + +void +DeferredReadManager::delayRead(DeferredRead const &aRead) +{ + fatal ("Not implemented"); +} + +void +DeferredReadManager::kickReads(int const count) +{ + fatal ("Not implemented"); +} + +void +comm_read(int fd, char *buf, int size, IOCB *handler, void *handler_data) +{ + fatal ("Not implemented"); +} + +/* should be in stub_CommRead */ +#include "CommRead.h" +CommRead::CommRead (int fd, char *buf, int len, IOCB *handler, void *data) +{ + fatal ("Not implemented"); +} + +CommRead::CommRead () +{ + fatal ("Not implemented"); +} + +void +commSetCloseOnExec(int fd) +{ + /* for tests... ignore */ +} + +void +commSetSelect(int fd, unsigned int type, PF * handler, void *client_data, + time_t timeout) +{ + /* all test code runs synchronously at the moment */ +} + +int +ignoreErrno(int ierrno) +{ + fatal ("Not implemented"); + return -1; +} + +int +commSetTimeout(int fd, int timeout, PF * handler, void *data) +{ + fatal ("Not implemented"); + return -1; +} + +int +commUnsetNonBlocking(int fd) +{ + fatal ("Not implemented"); + return -1; +} + +/* bah, cheating on stub count */ + +int +ipcCreate(int type, const char *prog, const char *const args[], const char *name, int *rfd, int *wfd) +{ + fatal ("Not implemented"); + return -1; +} + +void +comm_init(void) +{ + fd_table =(fde *) xcalloc(Squid_MaxFD, sizeof(fde)); + + /* Keep a few file descriptors free so that we don't run out of FD's + * after accepting a client but before it opens a socket or a file. + * Since Squid_MaxFD can be as high as several thousand, don't waste them */ + RESERVED_FD = XMIN(100, Squid_MaxFD / 4); +} + diff --git a/src/tests/stub_fd.cc b/src/tests/stub_fd.cc new file mode 100644 index 0000000000..46ec376a0a --- /dev/null +++ b/src/tests/stub_fd.cc @@ -0,0 +1,66 @@ +/* + * $Id: stub_fd.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +int +fdNFree(void) +{ + fatal ("Not Implemented"); + return -1; +} + +void +fd_open(int fd, unsigned int type, const char *desc) +{ + fatal ("Not Implemented"); +} + +void +fd_close(int fd) +{ + fatal ("Not Implemented"); +} + +void +fd_bytes(int fd, int len, unsigned int type) +{ + fatal ("Not Implemented"); +} + +void +fd_note(int fd, const char *s) +{ + fatal ("Not Implemented"); +} diff --git a/src/tests/stub_http.cc b/src/tests/stub_http.cc new file mode 100644 index 0000000000..39137d5c3c --- /dev/null +++ b/src/tests/stub_http.cc @@ -0,0 +1,42 @@ +/* + * $Id: stub_http.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +const char * +httpMakeVaryMark(HttpRequest * request, HttpReply const * reply) +{ + fatal ("Not implemented"); + return NULL; +} diff --git a/src/tests/stub_internal.cc b/src/tests/stub_internal.cc new file mode 100644 index 0000000000..f3372a0830 --- /dev/null +++ b/src/tests/stub_internal.cc @@ -0,0 +1,42 @@ +/* + * $Id: stub_internal.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +char * +internalLocalUri(const char *dir, const char *name) +{ + fatal ("Not implemented"); + return NULL; +} diff --git a/src/tests/stub_mem_node.cc b/src/tests/stub_mem_node.cc new file mode 100644 index 0000000000..f56d0bb1fa --- /dev/null +++ b/src/tests/stub_mem_node.cc @@ -0,0 +1,48 @@ +/* + * $Id: stub_mem_node.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "mem_node.h" + +mem_node::mem_node(off_t offset):nodeBuffer(0,offset,data) +{ + fatal ("Not implemented"); +} + +size_t +mem_node::InUseCount() +{ + fatal ("Not implemented"); + return 0; +} diff --git a/src/tests/stub_stmem.cc b/src/tests/stub_stmem.cc new file mode 100644 index 0000000000..443607b2cf --- /dev/null +++ b/src/tests/stub_stmem.cc @@ -0,0 +1,49 @@ +/* + * $Id: stub_stmem.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "stmem.h" + +mem_hdr::mem_hdr() +{} + +mem_hdr::~mem_hdr() +{} + +size_t +mem_hdr::size() const +{ + fatal ("Not implemented"); + return 0; +} diff --git a/src/tests/stub_store_client.cc b/src/tests/stub_store_client.cc new file mode 100644 index 0000000000..12eb7de850 --- /dev/null +++ b/src/tests/stub_store_client.cc @@ -0,0 +1,107 @@ +/* + * $Id: stub_store_client.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "StoreClient.h" + +int +storePendingNClients(const StoreEntry * e) +{ + /* no clients in the tests so far */ + return 0; +} + +/* Garh, too many stub files */ + +void +InvokeHandlers(StoreEntry * e) +{} + +void +storeLog(int tag, const StoreEntry * e) +{ + /* do nothing for tests - we don't need the log */ +} + +void +storeLogOpen(void) +{ + fatal ("Not implemented"); +} + +void +storeDigestInit(void) +{ + fatal ("Not implemented"); +} + +void +storeRebuildStart(void) +{ + fatal ("Not implemented"); +} + +#include "Store.h" +const char * +storeEntryFlags(const StoreEntry *) +{ + fatal ("Not implemented"); + return NULL; +} + +void +storeReplSetup(void) +{ + fatal ("Not implemented"); +} + +bool +store_client::memReaderHasLowerOffset(off_t anOffset) const +{ + fatal ("Not implemented"); + return false; +} + +void +store_client::dumpStats(MemBuf * output, int clientNumber) const +{ + fatal ("Not implemented"); +} + +int +store_client::getType() const +{ + return type; +} + diff --git a/src/tests/stub_store_rebuild.cc b/src/tests/stub_store_rebuild.cc new file mode 100644 index 0000000000..10ac8440ec --- /dev/null +++ b/src/tests/stub_store_rebuild.cc @@ -0,0 +1,47 @@ +/* + * $Id: stub_store_rebuild.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +void +storeRebuildProgress(int sd_index, int total, int sofar) +{ + fatal ("Not implemented"); +} + +void + +storeRebuildComplete(struct _store_rebuild_data *dc) +{} + diff --git a/src/tests/stub_store_swapout.cc b/src/tests/stub_store_swapout.cc new file mode 100644 index 0000000000..53256be273 --- /dev/null +++ b/src/tests/stub_store_swapout.cc @@ -0,0 +1,86 @@ +/* + * $Id: stub_store_swapout.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * DEBUG: section 84 Helper process maintenance + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +void +storeSwapOutFileClose(StoreEntry * e) +{ + fatal ("Not implemented"); +} + +int +storeSwapOutAble(const StoreEntry * e) +{ + fatal ("Not implemented"); + return -1; +} + +/* wrong stub file... */ +void +storeUnlink(StoreEntry * e) +{ + fatal ("Not implemented"); +} + + +void +storeIOStats(StoreEntry *) +{ + fatal ("Not implemented"); +} + + +#include "StoreMeta.h" + +char * +storeSwapMetaPack(tlv * tlv_list, int *length) +{ + fatal ("Not implemented"); + return NULL; +} + +tlv * +storeSwapMetaBuild(StoreEntry * e) +{ + fatal ("Not implemented"); + return NULL; +} + +void +storeSwapTLVFree(tlv * n) +{ + fatal ("Not implemented"); +} + diff --git a/src/tests/stub_tools.cc b/src/tests/stub_tools.cc new file mode 100644 index 0000000000..be6a7dce97 --- /dev/null +++ b/src/tests/stub_tools.cc @@ -0,0 +1,56 @@ +/* + * $Id: stub_tools.cc,v 1.1 2005/01/03 16:08:27 robertc Exp $ + * + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" + +time_t +getCurrentTime(void) +{ +#if GETTIMEOFDAY_NO_TZP + gettimeofday(¤t_time); +#else + + gettimeofday(¤t_time, NULL); +#endif + + current_dtime = (double) current_time.tv_sec + + (double) current_time.tv_usec / 1000000.0; + return squid_curtime = current_time.tv_sec; +} + +int +percent(int a, int b) +{ + return b ? ((int) (100.0 * a / b + 0.5)) : 0; +} + diff --git a/src/tests/testCoss.cc b/src/tests/testCoss.cc new file mode 100644 index 0000000000..ee49993e12 --- /dev/null +++ b/src/tests/testCoss.cc @@ -0,0 +1,264 @@ +#include "config.h" +#include + +#include "testCoss.h" +#include "squid.h" +#include "Store.h" +#include "SwapDir.h" +#include "DiskIO/DiskIOModule.h" +#include "fs/ufs/ufscommon.h" +#include "fs/coss/CossSwapDir.h" +#include "Mem.h" +#include "HttpHeader.h" +#include "HttpReply.h" +#include "StoreFileSystem.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testCoss ); + +typedef RefCount SwapDirPointer; +extern REMOVALPOLICYCREATE createRemovalPolicy_lru; + +static void +addSwapDir(SwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + +void +testCoss::commonInit() +{ + static bool inited = false; + + if (inited) + return; + + StoreFileSystem::SetupAllFs(); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; + + Config.store_dir_select_algorithm = xstrdup("round-robin"); + + Config.replPolicy = new RemovalPolicySettings; + + Config.replPolicy->type = xstrdup ("lru"); + + Config.replPolicy->args = NULL; + + /* garh garh */ + storeReplAdd("lru", createRemovalPolicy_lru); + + Mem::Init(); + + cbdataInit(); + + eventInit(); /* eventInit() is required for config parsing */ + + comm_init(); + + httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ + + httpReplyInitModule(); /* must go before accepting replies */ + + mem_policy = createRemovalPolicy(Config.replPolicy); + + inited = true; +} + +void +testCoss::testCossCreate() +{ + if (0 > system ("rm -rf testCoss::testCossSearch")) + throw std::runtime_error("Failed to clean test work directory"); + + StorePointer aRoot (new StoreController); + + Store::Root(aRoot); + + SwapDirPointer aStore (new CossSwapDir()); + + addSwapDir(aStore); + + commonInit(); + + char *path=xstrdup("testCoss::testCossSearch"); + + char *config_line=xstrdup("foo 100 max-size=102400 block-size=512 IOEngine=Blocking"); + + strtok(config_line, w_space); + + aStore->parse(0, path); + + safe_free(path); + + safe_free(config_line); + + /* ok, ready to create */ + aStore->create(); + + struct stat sb; + + CPPUNIT_ASSERT(::stat("testCoss::testCossSearch", &sb) == 0); + + /* TODO: check the size */ + + free_cachedir(&Config.cacheSwap); + + Store::Root(NULL); + + /* todo: here we should test a dirty rebuild */ + + // safe_free(Config.replPolicy->type); + // delete Config.replPolicy; + if (0 > system ("rm -rf testCoss::testCossSearch")) + throw std::runtime_error("Failed to clean test work directory"); +} + +/* TODO make this a cbdata class */ + +static bool cbcalled; + +static void +searchCallback(void *cbdata) +{ + cbcalled = true; +} + +void +testCoss::testCossSearch() +{ + /* test sequence + * make a valid working ufs swapdir + * put two entries in it and sync logs + * search the ufs dir + * check the entries we find are what we want + */ + + if (0 > system ("rm -rf testCoss::testCossSearch")) + throw std::runtime_error("Failed to clean test work directory"); + + StorePointer aRoot (new StoreController); + + Store::Root(aRoot); + + SwapDirPointer aStore (new CossSwapDir()); + + addSwapDir(aStore); + + commonInit(); + + char *path=xstrdup("testCoss::testCossSearch"); + + char *config_line=xstrdup("foo 100 max-size=102400 block-size=512 IOEngine=Blocking"); + + strtok(config_line, w_space); + + aStore->parse(0, path); + + safe_free(path); + + safe_free(config_line); + + /* ok, ready to create */ + aStore->create(); + + /* ok, ready to use */ + Store::Root().init(); + + /* ensure rebuilding finishes */ + while (store_dirs_rebuilding > 1) { + getCurrentTime(); + eventRun(); + } + + /* nothing to rebuild */ + CPPUNIT_ASSERT(store_dirs_rebuilding == 1); + + --store_dirs_rebuilding; + + /* add an entry */ + { + /* Create "vary" base object */ + request_flags flags; + flags.cachable = 1; + StoreEntry *pe = storeCreateEntry("dummy url", "dummy log url", flags, METHOD_GET); + HttpVersion version(1, 0); + /* We are allowed to do this typecast */ + httpReplySetHeaders((HttpReply *)pe->getReply(), version, HTTP_OK, "dummy test object", "x-squid-internal/test", -1, -1, squid_curtime + 100000); + + storeSetPublicKey(pe); + + storeBuffer(pe); + /* TODO: remove this when the metadata is separated */ + { + Packer p; + packerToStoreInit(&p, pe); + httpReplyPackHeadersInto(pe->getReply(), &p); + packerClean(&p); + } + + storeBufferFlush(pe); + storeTimestampsSet(pe); + pe->complete(); + storeSwapOut(pe); + CPPUNIT_ASSERT(pe->swap_dirn == 0); + CPPUNIT_ASSERT(pe->swap_filen == 0); + storeUnlockObject(pe); + } + + storeDirWriteCleanLogs(0); + + /* here we cheat: we know that UFSSwapDirs search off disk. If we did an init call to a new + * swapdir instance, we'd not be testing a clean build. + */ + StoreSearchPointer search = aStore->search (NULL, NULL); /* search for everything in the store */ + + /* nothing should be immediately available */ +#if 0 + + CPPUNIT_ASSERT(search->next() == false); +#endif + + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == NULL); + + /* trigger a callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + //CPPUNIT_ASSERT(search->next() == false); + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() != NULL); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* now we should have no error, we should have finished and have no current item */ + //CPPUNIT_ASSERT(search->next() == false); + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + + free_cachedir(&Config.cacheSwap); + + Store::Root(NULL); + + /* todo: here we should test a dirty rebuild */ + + //TODO: do this once, or each time. safe_free(Config.replPolicy->type); + // delete Config.replPolicy; + + if (0 > system ("rm -rf testCoss::testCossSearch")) + throw std::runtime_error("Failed to clean test work directory"); +} diff --git a/src/tests/testCoss.h b/src/tests/testCoss.h new file mode 100644 index 0000000000..5eaadf1cc6 --- /dev/null +++ b/src/tests/testCoss.h @@ -0,0 +1,27 @@ + +#ifndef SQUID_SRC_TEST_STORECONTROLLER_H +#define SQUID_SRC_TEST_STORECONTROLLER_H + +#include + +/* + * test the store framework + */ + +class testCoss : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testCoss ); + CPPUNIT_TEST( testCossCreate ); + CPPUNIT_TEST( testCossSearch ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void commonInit(); + void testCossCreate(); + void testCossSearch(); +}; + +#endif + diff --git a/src/tests/testNull.cc b/src/tests/testNull.cc new file mode 100644 index 0000000000..6ec350776b --- /dev/null +++ b/src/tests/testNull.cc @@ -0,0 +1,212 @@ +#include "config.h" +#include + +#include "testNull.h" +#include "squid.h" +#include "Store.h" +#include "SwapDir.h" +#include "DiskIO/DiskIOModule.h" +#include "fs/ufs/ufscommon.h" +#include "fs/null/store_null.h" +#include "Mem.h" +#include "HttpHeader.h" +#include "HttpReply.h" +#include "StoreFileSystem.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testNull ); + +typedef RefCount SwapDirPointer; +extern REMOVALPOLICYCREATE createRemovalPolicy_lru; + +static void +addSwapDir(SwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + +void +testNull::commonInit() +{ + static bool inited = false; + + if (inited) + return; + + StoreFileSystem::SetupAllFs(); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; + + Config.store_dir_select_algorithm = xstrdup("round-robin"); + + Config.replPolicy = new RemovalPolicySettings; + + Config.replPolicy->type = xstrdup ("lru"); + + Config.replPolicy->args = NULL; + + /* garh garh */ + storeReplAdd("lru", createRemovalPolicy_lru); + + Mem::Init(); + + cbdataInit(); + + eventInit(); /* eventInit() is required for config parsing */ + + comm_init(); + + httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ + + httpReplyInitModule(); /* must go before accepting replies */ + + mem_policy = createRemovalPolicy(Config.replPolicy); + + inited = true; +} + +void +testNull::testNullCreate() +{ + StorePointer aRoot (new StoreController); + Store::Root(aRoot); + SwapDirPointer aStore (new NullSwapDir()); + addSwapDir(aStore); + + commonInit(); + + char *path=xstrdup("testNull::testNullSearch"); + char *config_line=xstrdup("foo"); + strtok(config_line, w_space); + aStore->parse(0, path); + safe_free(path); + safe_free(config_line); + + /* ok, ready to create */ + aStore->create(); + + free_cachedir(&Config.cacheSwap); + Store::Root(NULL); + + /* todo: here we should test a dirty rebuild */ + + // safe_free(Config.replPolicy->type); + // delete Config.replPolicy; +} + +/* TODO make this a cbdata class */ + +static bool cbcalled; + +static void +searchCallback(void *cbdata) +{ + cbcalled = true; +} + +void +testNull::testNullSearch() +{ + /* test sequence + * make a valid working ufs swapdir + * put two entries in it and sync logs + * search the ufs dir + * check the entries we find are what we want + */ + StorePointer aRoot (new StoreController); + Store::Root(aRoot); + SwapDirPointer aStore (new NullSwapDir()); + addSwapDir(aStore); + + commonInit(); + + char *path=xstrdup("testNull::testNullSearch"); + char *config_line=xstrdup("foo"); + strtok(config_line, w_space); + aStore->parse(0, path); + safe_free(path); + safe_free(config_line); + + /* ok, ready to create */ + aStore->create(); + + /* ok, ready to use */ + Store::Root().init(); + + /* ensure rebuilding finishes */ + + while (store_dirs_rebuilding > 1) { + getCurrentTime(); + eventRun(); + } + + /* nothing to rebuild */ + CPPUNIT_ASSERT(store_dirs_rebuilding == 1); + + --store_dirs_rebuilding; + + /* add an entry */ + { + /* Create "vary" base object */ + request_flags flags; + flags.cachable = 1; + StoreEntry *pe = storeCreateEntry("dummy url", "dummy log url", flags, METHOD_GET); + HttpVersion version(1, 0); + /* We are allowed to do this typecast */ + httpReplySetHeaders((HttpReply *)pe->getReply(), version, HTTP_OK, "dummy test object", "x-squid-internal/test", -1, -1, squid_curtime + 100000); + + storeSetPublicKey(pe); + + storeBuffer(pe); + /* TODO: remove this when the metadata is separated */ + { + Packer p; + packerToStoreInit(&p, pe); + httpReplyPackHeadersInto(pe->getReply(), &p); + packerClean(&p); + } + + storeBufferFlush(pe); + storeTimestampsSet(pe); + pe->complete(); + storeSwapOut(pe); + /* Null does not accept store entries */ + CPPUNIT_ASSERT(pe->swap_dirn == -1); + storeUnlockObject(pe); + } + + storeDirWriteCleanLogs(0); + + /* here we cheat: we know that UFSSwapDirs search off disk. If we did an init call to a new + * swapdir instance, we'd not be testing a clean build. + */ + StoreSearchPointer search = aStore->search (NULL, NULL); /* search for everything in the store */ + + /* nothing should be available */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + CPPUNIT_ASSERT(search->next() == false); + + /* trigger a callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* still nothing */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + CPPUNIT_ASSERT(search->next() == false); + + free_cachedir(&Config.cacheSwap); + Store::Root(NULL); + + //TODO: do this once, or each time. safe_free(Config.replPolicy->type); + // delete Config.replPolicy; +} diff --git a/src/tests/testNull.h b/src/tests/testNull.h new file mode 100644 index 0000000000..76441c823e --- /dev/null +++ b/src/tests/testNull.h @@ -0,0 +1,27 @@ + +#ifndef SQUID_SRC_TEST_STORECONTROLLER_H +#define SQUID_SRC_TEST_STORECONTROLLER_H + +#include + +/* + * test the store framework + */ + +class testNull : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testNull ); + CPPUNIT_TEST( testNullCreate ); + CPPUNIT_TEST( testNullSearch ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void commonInit(); + void testNullCreate(); + void testNullSearch(); +}; + +#endif + diff --git a/src/tests/testStore.cc b/src/tests/testStore.cc new file mode 100644 index 0000000000..bc78a90086 --- /dev/null +++ b/src/tests/testStore.cc @@ -0,0 +1,134 @@ +#include "config.h" + +#include "testStore.h" +#include "squid.h" +#include "Store.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testStore ); + +class TestStore : public Store +{ + +public: + TestStore() : statsCalled (false) {} + + bool statsCalled; + + virtual int callback(); + + virtual StoreEntry* get + (const cache_key*); + + virtual void get + (String, void (*)(StoreEntry*, void*), void*); + + virtual void init(); + +virtual void maintain() {}; + + virtual size_t maxSize() const; + + virtual size_t minSize() const; + + virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */ + + virtual void reference(StoreEntry &){} /* Reference this object */ + + virtual void dereference(StoreEntry &){} /* Unreference this object */ + + virtual void updateSize(size_t size, int sign) {} + + virtual StoreSearch *search(String const url, HttpRequest *); +}; + +typedef RefCount TestStorePointer; + +int +TestStore::callback() +{ + return 1; +} + +StoreEntry* + +TestStore::get + (const cache_key*) +{ + return NULL; +} + +void + +TestStore::get + (String, void (*)(StoreEntry*, void*), void*) +{} + +void + +TestStore::init() +{} + +size_t +TestStore::maxSize() const +{ + return 3; +} + +size_t +TestStore::minSize() const +{ + return 1; +} + +void +TestStore::stat(StoreEntry &) const +{ + const_cast(this)->statsCalled = true; +} + +StoreSearch * +TestStore::search(String const url, HttpRequest *) +{ + return NULL; +} + +void +testStore::testSetRoot() +{ + StorePointer aStore (new TestStore); + Store::Root(aStore); + + CPPUNIT_ASSERT(&Store::Root() == aStore.getRaw()); + Store::Root(NULL); +} + +void +testStore::testUnsetRoot() +{ + StorePointer aStore (new TestStore); + StorePointer aStore2 (new TestStore); + Store::Root(aStore); + Store::Root(aStore2); + CPPUNIT_ASSERT(&Store::Root() == aStore2.getRaw()); + Store::Root(NULL); +} + +void +testStore::testStats() +{ + TestStorePointer aStore (new TestStore); + Store::Root(aStore.getRaw()); + CPPUNIT_ASSERT(aStore->statsCalled == false); + Store::Stats(NullStoreEntry::getInstance()); + CPPUNIT_ASSERT(aStore->statsCalled == true); + Store::Root(NULL); +} + +void +testStore::testMaxSize() +{ + StorePointer aStore (new TestStore); + Store::Root(aStore.getRaw()); + CPPUNIT_ASSERT(aStore->maxSize() == 3); + Store::Root(NULL); +} diff --git a/src/tests/testStore.h b/src/tests/testStore.h new file mode 100644 index 0000000000..dcf4a38d60 --- /dev/null +++ b/src/tests/testStore.h @@ -0,0 +1,30 @@ + +#ifndef SQUID_SRC_TEST_STORE_H +#define SQUID_SRC_TEST_STORE_H + +#include + +/* + * test the store framework + */ + +class testStore : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testStore ); + CPPUNIT_TEST( testSetRoot ); + CPPUNIT_TEST( testUnsetRoot ); + CPPUNIT_TEST( testStats ); + CPPUNIT_TEST( testMaxSize ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void testSetRoot(); + void testUnsetRoot(); + void testStats(); + void testMaxSize(); +}; + +#endif + diff --git a/src/tests/testStoreController.cc b/src/tests/testStoreController.cc new file mode 100644 index 0000000000..adf3aa678a --- /dev/null +++ b/src/tests/testStoreController.cc @@ -0,0 +1,187 @@ +#include "config.h" + +#include "testStoreController.h" +#include "squid.h" +#include "Store.h" +#include "SwapDir.h" +#include "TestSwapDir.h" +#include "Mem.h" +#include "StoreSearch.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testStoreController ); + +static void +addSwapDir(TestSwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + + +void +testStoreController::testStats() +{ + StoreEntry * logEntry = new StoreEntry("dummy_url", "dummy_log_url"); + logEntry->store_status = STORE_PENDING; + StorePointer aRoot (new StoreController); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + CPPUNIT_ASSERT(aStore->statsCalled == false); + CPPUNIT_ASSERT(aStore2->statsCalled == false); + Store::Stats(logEntry); + free_cachedir(&Config.cacheSwap); + CPPUNIT_ASSERT(aStore->statsCalled == true); + CPPUNIT_ASSERT(aStore2->statsCalled == true); + Store::Root(NULL); +} + +static void +commonInit() +{ + static bool inited = false; + + if (inited) + return; + + Mem::Init(); + + cbdataInit(); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; + + Config.store_dir_select_algorithm = xstrdup("round-robin"); +} + +void +testStoreController::testMaxSize() +{ + commonInit(); + StoreEntry * logEntry = new StoreEntry("dummy_url", "dummy_log_url"); + logEntry->store_status = STORE_PENDING; + StorePointer aRoot (new StoreController); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + CPPUNIT_ASSERT(Store::Root().maxSize() == 6); + free_cachedir(&Config.cacheSwap); + Store::Root(NULL); +} + +static StoreEntry * +addedEntry(StorePointer hashStore, + StorePointer aStore, + String name, + String varySpec, + String varyKey + + ) +{ + StoreEntry *e = new StoreEntry(); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; /* bogus haha */ + e->swap_filen = 0; /* garh - lower level*/ + e->swap_dirn = -1; + + for (int i=0; i < Config.cacheSwap.n_configured; i++) { + if (INDEXSD (i) == aStore.getRaw()) + e->swap_dirn = i; + } + + CPPUNIT_ASSERT (e->swap_dirn != -1); + e->swap_file_sz = 0; /* garh lower level */ + e->lock_count = 0; + e->lastref = squid_curtime; + e->timestamp = squid_curtime; + e->expires = squid_curtime; + e->lastmod = squid_curtime; + e->refcount = 1; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeHashInsert(e, (const cache_key *)name.buf()); /* do it after we clear KEY_PRIVATE */ + return e; +} + +/* TODO make this a cbdata class */ + +static bool cbcalled; + +static void +searchCallback(void *cbdata) +{ + cbcalled = true; +} + +void +testStoreController::testSearch() +{ + commonInit(); + StorePointer aRoot (new StoreController()); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + Store::Root().init(); + StoreEntry * entry1 = addedEntry (&Store::Root(), aStore.getRaw(), "name", NULL, NULL); + StoreEntry * entry2 = addedEntry (&Store::Root(), aStore2.getRaw(), "name2", NULL, NULL); + StoreSearchPointer search = aRoot->search (NULL, NULL); /* search for everything in the store */ + + /* nothing should be immediately available */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == NULL); +#if 0 + + CPPUNIT_ASSERT(search->next() == false); +#endif + + /* trigger a callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + /* note the hash order is random - the test happens to be in a nice order */ + CPPUNIT_ASSERT(search->currentItem() == entry1); + //CPPUNIT_ASSERT(search->next() == false); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == entry2); + //CPPUNIT_ASSERT(search->next() == false); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* now we should have no error, we should have finished and have no current item */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + //CPPUNIT_ASSERT(search->next() == false); + + Store::Root(NULL); +} diff --git a/src/tests/testStoreController.h b/src/tests/testStoreController.h new file mode 100644 index 0000000000..85f57ef05f --- /dev/null +++ b/src/tests/testStoreController.h @@ -0,0 +1,28 @@ + +#ifndef SQUID_SRC_TEST_STORECONTROLLER_H +#define SQUID_SRC_TEST_STORECONTROLLER_H + +#include + +/* + * test the store framework + */ + +class testStoreController : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testStoreController ); + CPPUNIT_TEST( testStats ); + CPPUNIT_TEST( testMaxSize ); + CPPUNIT_TEST( testSearch ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void testStats(); + void testMaxSize(); + void testSearch(); +}; + +#endif + diff --git a/src/tests/testStoreHashIndex.cc b/src/tests/testStoreHashIndex.cc new file mode 100644 index 0000000000..b599cd7dd1 --- /dev/null +++ b/src/tests/testStoreHashIndex.cc @@ -0,0 +1,182 @@ +#include "config.h" + +#include "testStoreHashIndex.h" +#include "squid.h" +#include "Store.h" +#include "SwapDir.h" +#include "TestSwapDir.h" +#include "StoreHashIndex.h" +#include "Mem.h" +#include "StoreSearch.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testStoreHashIndex ); + +static void +addSwapDir(TestSwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + + +void +testStoreHashIndex::testStats() +{ + StoreEntry * logEntry = new StoreEntry("dummy_url", "dummy_log_url"); + logEntry->store_status = STORE_PENDING; + StorePointer aRoot (new StoreHashIndex()); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + CPPUNIT_ASSERT(aStore->statsCalled == false); + CPPUNIT_ASSERT(aStore2->statsCalled == false); + Store::Stats(logEntry); + free_cachedir(&Config.cacheSwap); + CPPUNIT_ASSERT(aStore->statsCalled == true); + CPPUNIT_ASSERT(aStore2->statsCalled == true); + Store::Root(NULL); +} + +void +testStoreHashIndex::testMaxSize() +{ + StoreEntry * logEntry = new StoreEntry("dummy_url", "dummy_log_url"); + logEntry->store_status = STORE_PENDING; + StorePointer aRoot (new StoreHashIndex()); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + CPPUNIT_ASSERT(Store::Root().maxSize() == 6); + free_cachedir(&Config.cacheSwap); + Store::Root(NULL); +} + +StoreEntry * +addedEntry(StorePointer hashStore, + StorePointer aStore, + String name, + String varySpec, + String varyKey + + ) +{ + StoreEntry *e = new StoreEntry(); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; /* bogus haha */ + e->swap_filen = 0; /* garh - lower level*/ + e->swap_dirn = -1; + + for (int i=0; i < Config.cacheSwap.n_configured; i++) { + if (INDEXSD (i) == aStore.getRaw()) + e->swap_dirn = i; + } + + CPPUNIT_ASSERT (e->swap_dirn != -1); + e->swap_file_sz = 0; /* garh lower level */ + e->lock_count = 0; + e->lastref = squid_curtime; + e->timestamp = squid_curtime; + e->expires = squid_curtime; + e->lastmod = squid_curtime; + e->refcount = 1; + EBIT_SET(e->flags, ENTRY_CACHABLE); + EBIT_CLR(e->flags, RELEASE_REQUEST); + EBIT_CLR(e->flags, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flags, ENTRY_VALIDATED); + storeHashInsert(e, (const cache_key *)name.buf()); /* do it after we clear KEY_PRIVATE */ + return e; +} + +void commonInit() +{ + static bool inited = false; + + if (inited) + return; + + Mem::Init(); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; +} + +/* TODO make this a cbdata class */ + +static bool cbcalled; + +static void +searchCallback(void *cbdata) +{ + cbcalled = true; +} + +void +testStoreHashIndex::testSearch() +{ + commonInit(); + StorePointer aRoot (new StoreHashIndex()); + Store::Root(aRoot); + TestSwapDirPointer aStore (new TestSwapDir); + TestSwapDirPointer aStore2 (new TestSwapDir); + addSwapDir(aStore); + addSwapDir(aStore2); + Store::Root().init(); + StoreEntry * entry1 = addedEntry (&Store::Root(), aStore.getRaw(), "name", NULL, NULL); + StoreEntry * entry2 = addedEntry (&Store::Root(), aStore2.getRaw(), "name2", NULL, NULL); + StoreSearchPointer search = aRoot->search (NULL, NULL); /* search for everything in the store */ + + /* nothing should be immediately available */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == NULL); +#if 0 + + CPPUNIT_ASSERT(search->next() == false); +#endif + + /* trigger a callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + /* note the hash order is random - the test happens to be in a nice order */ + CPPUNIT_ASSERT(search->currentItem() == entry1); + //CPPUNIT_ASSERT(search->next() == false); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == entry2); + //CPPUNIT_ASSERT(search->next() == false); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* now we should have no error, we should have finished and have no current item */ + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + //CPPUNIT_ASSERT(search->next() == false); + + Store::Root(NULL); +} diff --git a/src/tests/testStoreHashIndex.h b/src/tests/testStoreHashIndex.h new file mode 100644 index 0000000000..af940e6a37 --- /dev/null +++ b/src/tests/testStoreHashIndex.h @@ -0,0 +1,28 @@ + +#ifndef SQUID_SRC_TEST_STOREHASHINDEX_H +#define SQUID_SRC_TEST_STOREHASHINDEX_H + +#include + +/* + * test the store framework + */ + +class testStoreHashIndex : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testStoreHashIndex ); + CPPUNIT_TEST( testStats ); + CPPUNIT_TEST( testMaxSize ); + CPPUNIT_TEST( testSearch ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void testStats(); + void testMaxSize(); + void testSearch(); +}; + +#endif + diff --git a/src/tests/testUfs.cc b/src/tests/testUfs.cc new file mode 100644 index 0000000000..adb318a62e --- /dev/null +++ b/src/tests/testUfs.cc @@ -0,0 +1,196 @@ +#include "config.h" +#include + +#include "testUfs.h" +#include "squid.h" +#include "Store.h" +#include "SwapDir.h" +#include "DiskIO/DiskIOModule.h" +#include "fs/ufs/ufscommon.h" +#include "Mem.h" +#include "HttpHeader.h" +#include "HttpReply.h" + +CPPUNIT_TEST_SUITE_REGISTRATION( testUfs ); + +typedef RefCount SwapDirPointer; +extern REMOVALPOLICYCREATE createRemovalPolicy_lru; + +static void +addSwapDir(SwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + +/* TODO make this a cbdata class */ + +static bool cbcalled; + +static void +searchCallback(void *cbdata) +{ + cbcalled = true; +} + +void +testUfs::testUfsSearch() +{ + /* test sequence + * make a valid working ufs swapdir + * put two entries in it and sync logs + * search the ufs dir + * check the entries we find are what we want + */ + + if (0 > system ("rm -rf testUfs::testUfsSearch")) + throw std::runtime_error("Failed to clean test work directory"); + + StorePointer aRoot (new StoreController); + + Store::Root(aRoot); + + SwapDirPointer aStore (new UFSSwapDir("ufs", "Blocking")); + + aStore->IO = new UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy()); + + addSwapDir(aStore); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; + + Config.store_dir_select_algorithm = xstrdup("round-robin"); + + Config.replPolicy = new RemovalPolicySettings; + + Config.replPolicy->type = xstrdup ("lru"); + + /* garh garh */ + storeReplAdd("lru", createRemovalPolicy_lru); + + Mem::Init(); + + cbdataInit(); + + eventInit(); /* eventInit() is required for config parsing */ + + comm_init(); + + httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ + + httpReplyInitModule(); /* must go before accepting replies */ + + mem_policy = createRemovalPolicy(Config.replPolicy); + + char *path=xstrdup("testUfs::testUfsSearch"); + + char *config_line=xstrdup("foo 100 1 1"); + + strtok(config_line, w_space); + + aStore->parse(0, path); + + safe_free(path); + + safe_free(config_line); + + /* ok, ready to create */ + aStore->create(); + + /* ok, ready to use - init store & hash too */ + Store::Root().init(); + + /* ensure rebuilding finishes */ + while (store_dirs_rebuilding > 1) { + getCurrentTime(); + eventRun(); + } + + /* nothing to rebuild */ + CPPUNIT_ASSERT(store_dirs_rebuilding == 1); + + --store_dirs_rebuilding; + + /* add an entry */ + { + /* Create "vary" base object */ + request_flags flags; + flags.cachable = 1; + StoreEntry *pe = storeCreateEntry("dummy url", "dummy log url", flags, METHOD_GET); + HttpVersion version(1, 0); + /* We are allowed to do this typecast */ + httpReplySetHeaders((HttpReply *)pe->getReply(), version, HTTP_OK, "dummy test object", "x-squid-internal/test", -1, -1, squid_curtime + 100000); + + storeSetPublicKey(pe); + + storeBuffer(pe); + /* TODO: remove this when the metadata is separated */ + { + Packer p; + packerToStoreInit(&p, pe); + httpReplyPackHeadersInto(pe->getReply(), &p); + packerClean(&p); + } + + storeBufferFlush(pe); + storeTimestampsSet(pe); + pe->complete(); + storeSwapOut(pe); + CPPUNIT_ASSERT(pe->swap_dirn == 0); + CPPUNIT_ASSERT(pe->swap_filen == 0); + storeUnlockObject(pe); + } + + storeDirWriteCleanLogs(0); + + /* here we cheat: we know that UFSSwapDirs search off disk. If we did an init call to a new + * swapdir instance, we'd not be testing a clean build. + */ + StoreSearchPointer search = aStore->search (NULL, NULL); /* search for everything in the store */ + + /* nothing should be immediately available */ +#if 0 + + CPPUNIT_ASSERT(search->next() == false); +#endif + + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() == NULL); + + /* trigger a callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* we should have access to a entry now, that matches the entry we had before */ + //CPPUNIT_ASSERT(search->next() == false); + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == false); + CPPUNIT_ASSERT(search->currentItem() != NULL); + + /* trigger another callback */ + cbcalled = false; + search->next(searchCallback, NULL); + CPPUNIT_ASSERT(cbcalled == true); + + /* now we should have no error, we should have finished and have no current item */ + //CPPUNIT_ASSERT(search->next() == false); + CPPUNIT_ASSERT(search->error() == false); + CPPUNIT_ASSERT(search->isDone() == true); + CPPUNIT_ASSERT(search->currentItem() == NULL); + + free_cachedir(&Config.cacheSwap); + + /* todo: here we should test a dirty rebuild */ + + safe_free(Config.replPolicy->type); + delete Config.replPolicy; + + if (0 > system ("rm -rf testUfs::testUfsSearch")) + throw std::runtime_error("Failed to clean test work directory"); +} diff --git a/src/tests/testUfs.h b/src/tests/testUfs.h new file mode 100644 index 0000000000..3cd86f10c8 --- /dev/null +++ b/src/tests/testUfs.h @@ -0,0 +1,24 @@ + +#ifndef SQUID_SRC_TEST_STORECONTROLLER_H +#define SQUID_SRC_TEST_STORECONTROLLER_H + +#include + +/* + * test the store framework + */ + +class testUfs : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testUfs ); + CPPUNIT_TEST( testUfsSearch ); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void testUfsSearch(); +}; + +#endif + diff --git a/src/tools.cc b/src/tools.cc index cc0e07d2c5..6a5a98358b 100644 --- a/src/tools.cc +++ b/src/tools.cc @@ -1,6 +1,6 @@ /* - * $Id: tools.cc,v 1.247 2004/12/20 16:30:37 robertc Exp $ + * $Id: tools.cc,v 1.248 2005/01/03 16:08:26 robertc Exp $ * * DEBUG: section 21 Misc Functions * AUTHOR: Harvest Derived @@ -34,6 +34,7 @@ */ #include "squid.h" +#include "SwapDir.h" #include "fde.h" #define DEAD_MSG "\ @@ -630,15 +631,6 @@ uniqueHostname(void) return Config.uniqueHostname ? Config.uniqueHostname : getMyHostname(); } -void -safeunlink(const char *s, int quiet) -{ - statCounter.syscalls.disk.unlinks++; - - if (unlink(s) < 0 && !quiet) - debug(50, 1) ("safeunlink: Couldn't delete %s: %s\n", s, xstrerror()); -} - /* leave a privilegied section. (Give up any privilegies) * Routines that need privilegies can rap themselves in enter_suid() * and leave_suid() @@ -971,12 +963,6 @@ logsFlush(void) fflush(debug_log); } -const char * -checkNullString(const char *p) -{ - return p ? p : "(NULL)"; -} - dlink_node * dlinkNodeNew() { @@ -1086,64 +1072,6 @@ debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm) memBufClean(&mb); } -void -linklistPush(link_list ** L, void *p) -{ - link_list *l = (link_list *)memAllocate(MEM_LINK_LIST); - l->next = NULL; - l->ptr = p; - - while (*L) - L = &(*L)->next; - - *L = l; -} - -void * -linklistShift(link_list ** L) -{ - void *p; - link_list *l; - - if (NULL == *L) - return NULL; - - l = *L; - - p = l->ptr; - - *L = (*L)->next; - - memFree(l, MEM_LINK_LIST); - - return p; -} - -/* - * Same as rename(2) but complains if something goes wrong; - * the caller is responsible for handing and explaining the - * consequences of errors. - */ -int -xrename(const char *from, const char *to) -{ - debug(21, 2) ("xrename: renaming %s to %s\n", from, to); -#if defined (_SQUID_OS2_) || defined (_SQUID_WIN32_) - - remove - (to); - -#endif - - if (0 == rename(from, to)) - return 0; - - debug(21, errno == ENOENT ? 2 : 1) ("xrename: Cannot rename %s to %s: %s\n", - from, to, xstrerror()); - - return -1; -} - void parseEtcHosts(void) { diff --git a/src/typedefs.h b/src/typedefs.h index 6cecef275f..7da732bab8 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -1,6 +1,6 @@ /* - * $Id: typedefs.h,v 1.176 2004/12/20 16:30:37 robertc Exp $ + * $Id: typedefs.h,v 1.177 2005/01/03 16:08:26 robertc Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -211,8 +211,6 @@ typedef struct _StatCounters StatCounters; typedef struct _storeSwapLogData storeSwapLogData; -typedef struct _cacheSwap cacheSwap; - typedef struct _StatHist StatHist; typedef struct _ClientInfo ClientInfo; @@ -259,8 +257,6 @@ typedef struct _RemovalPolicyWalker RemovalPolicyWalker; typedef struct _RemovalPurgeWalker RemovalPurgeWalker; -typedef struct _RemovalPolicySettings RemovalPolicySettings; - #if SQUID_SNMP typedef variable_list *(oid_ParseFn) (variable_list *, snint *); diff --git a/test-suite/test_tools.cc b/test-suite/test_tools.cc index f3d6fef39a..815452e4f5 100644 --- a/test-suite/test_tools.cc +++ b/test-suite/test_tools.cc @@ -1,6 +1,6 @@ /* - * $Id: test_tools.cc,v 1.6 2004/08/30 05:12:33 robertc Exp $ + * $Id: test_tools.cc,v 1.7 2005/01/03 16:08:27 robertc Exp $ * * AUTHOR: Robert Collins * @@ -262,3 +262,11 @@ dlinkDelete(dlink_node * m, dlink_list * list) m->next = m->prev = NULL; } +Ctx +ctx_enter(const char *descr) { + return 0; +} + +void +ctx_exit(Ctx ctx) { +}