From 73656056cdcface25c015c755e9a3688acd404fc Mon Sep 17 00:00:00 2001 From: Amos Jeffries Date: Mon, 30 Sep 2013 06:30:50 -0600 Subject: [PATCH] Remove COSS This storage type has been superceded by Rock storage since 3.2. --- configure.ac | 26 - doc/debug-sections.txt | 2 - src/Makefile.am | 165 --- src/StoreFileSystem.h | 2 +- src/StoreSwapLogData.h | 2 +- src/cf.data.pre | 25 +- src/fs/Makefile.am | 17 +- src/fs/Module.cc | 13 - src/fs/coss/CossSwapDir.h | 146 -- src/fs/coss/StoreFScoss.cc | 132 -- src/fs/coss/StoreFScoss.h | 96 -- src/fs/coss/coss-notes.txt | 128 -- src/fs/coss/store_coss.h | 133 -- src/fs/coss/store_dir_coss.cc | 1203 ----------------- src/fs/coss/store_io_coss.cc | 605 --------- src/fs/rock/RockSwapDir.cc | 2 +- src/tests/testCoss.cc | 298 ---- src/tests/testCoss.h | 29 - src/tests/testDiskIO.cc | 5 - test-suite/Makefile.am | 2 + test-suite/buildtests/layer-01-minimal.opts | 1 - test-suite/buildtests/layer-02-maximus.opts | 1 - .../layer-04-noauth-everything.opts | 1 - 23 files changed, 8 insertions(+), 3026 deletions(-) delete mode 100644 src/fs/coss/CossSwapDir.h delete mode 100644 src/fs/coss/StoreFScoss.cc delete mode 100644 src/fs/coss/StoreFScoss.h delete mode 100644 src/fs/coss/coss-notes.txt delete mode 100644 src/fs/coss/store_coss.h delete mode 100644 src/fs/coss/store_dir_coss.cc delete mode 100644 src/fs/coss/store_io_coss.cc delete mode 100644 src/tests/testCoss.cc delete mode 100644 src/tests/testCoss.h diff --git a/configure.ac b/configure.ac index 60ff2c0b9b..5acdc06564 100644 --- a/configure.ac +++ b/configure.ac @@ -794,8 +794,6 @@ if test "x$squid_opt_enable_storeio" = "xauto"; then squid_opt_enable_storeio=yes AC_MSG_CHECKING([for available StoreIO modules]) SQUID_LOOK_FOR_MODULES([$srcdir/src/fs],[squid_storeio_module_candidates]) - # disable coss - squid_storeio_module_candidates=`echo $squid_storeio_module_candidates|sed 's/coss//'` AC_MSG_RESULT([$squid_storeio_module_candidates]) fi @@ -817,15 +815,6 @@ for fs in $squid_storeio_module_candidates none; do AC_MSG_ERROR([Storage module aufs requires DiskIO module: Blocking or DiskThreads]) fi ;; - coss) - AC_MSG_WARN([COSS Support is not stable yet in Squid-3. Please do not use.]) - if ! test "x$squid_disk_module_candidates_AIO" = "xyes"; then - AC_MSG_ERROR([COSS requires POSIX AIO which is not available.]) - fi - # Automake on MinGW needs explicit exe extension - # for STORE_TESTS substition - STORE_TESTS="$STORE_TESTS tests/testCoss$EXEEXT" - ;; rock) if test "x$squid_disk_module_candidates_IpcIo" != "xyes" -a \ "x$squid_disk_module_candidates_Blocking" != "xyes"; then @@ -843,7 +832,6 @@ dnl make sure that global FS objects are linked to the squid binary. AH_TEMPLATE(HAVE_FS_UFS, "Define to 1 if ufs filesystem module is build") AH_TEMPLATE(HAVE_FS_AUFS, "Define to 1 if aufs filesystem module is build") AH_TEMPLATE(HAVE_FS_DISKD, "Define to 1 if diskd filesystem module is build") -AH_TEMPLATE(HAVE_FS_COSS, "Define to 1 if coss filesystem module is build") AH_TEMPLATE(HAVE_FS_ROCK, "Define to 1 if rock filesystem module is build") @@ -1306,20 +1294,6 @@ SQUID_DEFINE_BOOL(USE_CACHE_DIGESTS,${enable_cache_digests:=no}, AC_MSG_NOTICE([Cache Digests enabled: $enable_cache_digests]) -dnl Size of COSS memory buffer -squid_opt_coss_membuf_size=1048576 -AC_ARG_WITH(coss-membuf-size, - AS_HELP_STRING([--with-coss-membuf-size=size], - [COSS membuf size (default $squid_opt_coss_membuf_size bytes)]), [ -case $withval in - [[0-9]]*) squid_opt_coss_membuf_size=$withval ;; - *) AC_MSG_ERROR([--with-coss-membuf-size expects a numeric argument]) ;; -esac -]) -AC_MSG_NOTICE([Setting COSS membuf size to $squid_opt_coss_membuf_size bytes]) -AC_DEFINE_UNQUOTED(COSS_MEMBUF_SZ, $squid_opt_coss_membuf_size, - [Default COSS membuf size]) - ################################ # check for netio plugin stuff # ################################ diff --git a/doc/debug-sections.txt b/doc/debug-sections.txt index 04933e2699..9e65512983 100644 --- a/doc/debug-sections.txt +++ b/doc/debug-sections.txt @@ -89,7 +89,6 @@ section 46 Access Log - Squid ICAP Logging section 46 Access Log - Squid format section 46 Access Log - Squid referer format section 46 Access Log - Squid useragent format -section 47 Store COSS Directory Routines section 47 Store Directory Routines section 48 Persistent Connections section 49 SNMP Interface @@ -128,7 +127,6 @@ section 78 DNS lookups; interacts with lib/rfc1035.c section 79 Disk IO Routines section 79 Squid-side DISKD I/O functions. section 79 Squid-side Disk I/O functions. -section 79 Storage Manager COSS Interface section 79 Storage Manager UFS Interface section 79 Disk IO Routines section 80 WCCP Support diff --git a/src/Makefile.am b/src/Makefile.am index 7be4f7aae3..e3a7583fdb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -195,7 +195,6 @@ EXTRA_PROGRAMS = \ dnsserver \ recv-announce \ tests/testUfs \ - tests/testCoss \ tests/testRock \ ufsdump @@ -3411,170 +3410,6 @@ tests_testRock_LDFLAGS = $(INCLUDES) $(LIBADD_DL) tests_testRock_DEPENDENCIES = \ $(SWAP_TEST_DS) -tests_testCoss_SOURCES = \ - tests/testCoss.cc \ - tests/testMain.cc \ - tests/testCoss.h \ - tests/stub_cache_manager.cc \ - client_db.h \ - tests/stub_client_db.cc \ - tests/stub_debug.cc \ - tests/stub_HelperChildConfig.cc \ - internal.h \ - tests/stub_internal.cc \ - tests/stub_ipc.cc \ - tests/stub_libeui.cc \ - tests/stub_pconn.cc \ - store_rebuild.h \ - tests/stub_store_rebuild.cc \ - tests/stub_store_stats.cc \ - fatal.h \ - tests/stub_fatal.cc \ - fd.h \ - fd.cc \ - fde.h \ - fde.cc \ - disk.h \ - disk.cc \ - FileMap.h \ - filemap.cc \ - HttpBody.h \ - HttpBody.cc \ - HttpReply.cc \ - int.h \ - int.cc \ - SquidList.h \ - SquidList.cc \ - MemObject.cc \ - StoreSwapLogData.cc \ - StoreIOState.cc \ - StoreMeta.cc \ - StoreMetaMD5.cc \ - StoreMetaSTD.cc \ - StoreMetaSTDLFS.cc \ - StoreMetaUnpacker.cc \ - StoreMetaURL.cc \ - StoreMetaVary.cc \ - StoreFileSystem.cc \ - store_io.cc \ - store_swapout.cc \ - store_swapmeta.cc \ - $(UNLINKDSOURCE) \ - $(WIN32_SOURCE) \ - event.cc \ - $(DELAY_POOL_SOURCE) \ - CacheDigest.h \ - CacheDigest.cc \ - ConfigParser.cc \ - EventLoop.cc \ - HttpMsg.cc \ - RemovalPolicy.cc \ - store_dir.cc \ - repl_modules.h \ - store.cc \ - HttpRequestMethod.cc \ - store_key_md5.h \ - store_key_md5.cc \ - Parsing.cc \ - ConfigOption.cc \ - SwapDir.cc \ - tests/stub_acl.cc \ - cache_cf.h \ - YesNoNone.h \ - tests/stub_cache_cf.cc \ - tests/stub_helper.cc \ - cbdata.cc \ - String.cc \ - tests/stub_client_side_request.cc \ - tests/stub_http.cc \ - mem_node.cc \ - stmem.cc \ - mime.h \ - tests/stub_mime.cc \ - HttpHeaderFieldInfo.h \ - HttpHeaderTools.h \ - HttpHeaderTools.cc \ - HttpHeader.h \ - HttpHeader.cc \ - Mem.h \ - mem.cc \ - ClientInfo.h \ - MemBuf.cc \ - HttpHdrContRange.cc \ - Packer.cc \ - HttpHeaderFieldStat.h \ - HttpHdrCc.h \ - HttpHdrCc.cc \ - HttpHdrCc.cci \ - HttpHdrSc.cc \ - HttpHdrScTarget.cc \ - url.cc \ - StatCounters.h \ - StatCounters.cc \ - StatHist.h \ - tests/stub_StatHist.cc \ - tests/stub_errorpage.cc \ - tests/stub_HttpRequest.cc \ - log/access_log.h \ - tests/stub_access_log.cc \ - refresh.h \ - refresh.cc \ - tests/stub_MemStore.cc \ - tests/stub_Port.cc \ - tests/stub_store_client.cc \ - tests/stub_store_stats.cc \ - tools.h \ - tests/stub_tools.cc \ - tests/stub_UdsOp.cc \ - tests/testStoreSupport.cc \ - tests/testStoreSupport.h \ - time.cc \ - URLScheme.cc \ - wordlist.h \ - wordlist.cc \ - $(DISKIO_SOURCE) - -nodist_tests_testCoss_SOURCES = \ - swap_log_op.cc \ - SquidMath.cc \ - SquidMath.h \ - $(TESTSOURCES) \ - $(DISKIO_GEN_SOURCE) -tests_testCoss_LDADD = \ - anyp/libanyp.la \ - libsquid.la \ - $(REGEXLIB) \ - $(AUTH_ACL_LIBS) \ - ident/libident.la \ - acl/libacls.la \ - acl/libstate.la \ - $(AUTH_LIBS) \ - acl/libapi.la \ - libsquid.la \ - comm/libcomm.la \ - ip/libip.la \ - fs/libfs.la \ - mgr/libmgr.la \ - $(REPL_OBJS) \ - $(DISK_LIBS) \ - $(DISK_OS_LIBS) \ - $(COMMON_LIBS) \ - $(SSL_LIBS) \ - acl/libapi.la \ - ipc/libipc.la \ - base/libbase.la \ - $(top_builddir)/lib/libmisccontainers.la \ - $(top_builddir)/lib/libmiscencoding.la \ - $(top_builddir)/lib/libmiscutil.la \ - $(SQUID_CPPUNIT_LIBS) \ - $(REGEXLIB) \ - $(SSLLIB) \ - $(COMPAT_LIB) \ - $(XTRA_LIBS) -tests_testCoss_LDFLAGS = $(LIBADD_DL) -tests_testCoss_DEPENDENCIES = \ - $(SWAP_TEST_DS) - ## Tests of the URL module. ## TODO: Trim this down once the insanity is over. tests_testURL_SOURCES = \ diff --git a/src/StoreFileSystem.h b/src/StoreFileSystem.h index a83b848415..81dd906bb6 100644 --- a/src/StoreFileSystem.h +++ b/src/StoreFileSystem.h @@ -70,7 +70,7 @@ * configure will take a list of storage types through the * --enable-store-io parameter. This parameter takes a list of * space seperated storage types. For example, - * --enable-store-io="ufs coss" . + * --enable-store-io="ufs aufs" . * \par * Each storage type must create an archive file diff --git a/src/StoreSwapLogData.h b/src/StoreSwapLogData.h index 8a58046ab0..ede911c9f7 100644 --- a/src/StoreSwapLogData.h +++ b/src/StoreSwapLogData.h @@ -99,7 +99,7 @@ operator <<(std::ostream &os, const SwapChecksum24 &sum) * \par * Defines the structure of a binary swap.state file entry for UFS stores. - * TODO: Move to fs/ufs (and remove from COSS). + * TODO: Move to fs/ufs * \note StoreSwapLogData entries are written in native machine byte order * They are not necessarily portable across architectures. diff --git a/src/cf.data.pre b/src/cf.data.pre index e5d357722f..7e3e831fd9 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -3451,25 +3451,6 @@ DOC_START enforcement. Currently supported by IpcIo module only. - ==== The coss store type ==== - - NP: COSS filesystem in Squid-3 has been deemed too unstable for - production use and has thus been removed from this release. - We hope that it can be made usable again soon. - - block-size=n defines the "block size" for COSS cache_dir's. - Squid uses file numbers as block numbers. Since file numbers - are limited to 24 bits, the block size determines the maximum - size of the COSS partition. The default is 512 bytes, which - leads to a maximum cache_dir size of 512<<24, or 8 GB. Note - you should not change the coss block size after Squid - has written some objects to the cache_dir. - - The coss file store has changed from 2.5. Now it uses a file - called 'stripe' in the directory names in the config - and - this will be created by squid -z. - - ==== COMMON OPTIONS ==== no-store no new objects should be stored to this cache_dir. @@ -3478,7 +3459,7 @@ DOC_START will accept. It's used to restrict a cache_dir to only store large objects (e.g. AUFS) while other stores are optimized for smaller objects - (e.g. COSS). + (e.g. Rock). Defaults to 0. max-size=n the maximum object size in bytes this cache_dir @@ -3490,10 +3471,6 @@ DOC_START Note: To make optimal use of the max-size limits you should order the cache_dir lines with the smallest max-size value first. - Note for coss, max-size must be less than COSS_MEMBUF_SZ, - which can be changed with the --with-coss-membuf-size=N configure - option. - NOCOMMENT_START # Uncomment and adjust the following to add a disk cache directory. diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 444d35c505..247bc58b55 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -1,6 +1,6 @@ include $(top_srcdir)/src/Common.am -EXTRA_LTLIBRARIES = libaufs.la libdiskd.la libcoss.la libufs.la librock.la +EXTRA_LTLIBRARIES = libaufs.la libdiskd.la libufs.la librock.la noinst_LTLIBRARIES = $(STORE_LIBS_TO_BUILD) libfs.la # aufs is a "fake" legacy store @@ -11,14 +11,6 @@ libaufs_la_SOURCES = \ libdiskd_la_SOURCES = \ diskd/StoreFSdiskd.cc -libcoss_la_SOURCES = \ - coss/StoreFScoss.h \ - coss/StoreFScoss.cc \ - coss/store_coss.h \ - coss/store_io_coss.cc \ - coss/store_dir_coss.cc \ - coss/CossSwapDir.h - libufs_la_SOURCES = \ ufs/StoreFSufs.h \ ufs/StoreFSufs.cc \ @@ -52,16 +44,11 @@ libfs_la_SOURCES = Module.cc Module.h libfs_la_LIBADD = $(STORE_LIBS_TO_BUILD) libfs_la_DEPENDENCIES = $(STORE_LIBS_TO_BUILD) -EXTRA_DIST = \ - coss/coss-notes.txt - ## we need our local files too (but avoid -I. at all costs) INCLUDES += -I$(srcdir) ## targets below to emulate distributed makefiles -coss/all: libcoss.la -coss/clean: clean ufs/all: libufs.la ufs/clean: clean rock/all: librock.la @@ -72,7 +59,7 @@ TESTS += testHeaders ## Special Universal .h dependency test script ## aborts if error encountered -testHeaders: $(srcdir)/ufs/*.h $(srcdir)/coss/*.h $(srcdir)/rock/*.h +testHeaders: $(srcdir)/ufs/*.h $(srcdir)/rock/*.h $(SHELL) $(top_srcdir)/test-suite/testheaders.sh "$(CXXCOMPILE)" $^ || exit 1 ## diskd/ has no .h files ## aufs/ has no .h files diff --git a/src/fs/Module.cc b/src/fs/Module.cc index 86191b267a..146db3a2e9 100644 --- a/src/fs/Module.cc +++ b/src/fs/Module.cc @@ -5,10 +5,6 @@ #include "fs/ufs/UFSSwapDir.h" #endif -#if HAVE_FS_COSS -#include "fs/coss/StoreFScoss.h" -#endif - #if HAVE_FS_UFS static Fs::Ufs::StoreFSufs *UfsInstance; #endif @@ -26,15 +22,6 @@ static Fs::Ufs::StoreFSufs *DiskdInstance; static Rock::StoreFileSystem *RockInstance = NULL; #endif -/* TODO: Modify coss code to: - * (a) remove the StoreFScoss::GetInstance method, - * (b) declare the StoreFScoss::stats as static and - * (c) merge the StoreFScoss::stat() method with the static - * StoreFScoss::Stats() */ -#if HAVE_FS_COSS -StoreFScoss &CossInstance = StoreFScoss::GetInstance(); -#endif - void Fs::Init() { diff --git a/src/fs/coss/CossSwapDir.h b/src/fs/coss/CossSwapDir.h deleted file mode 100644 index dc0100e90e..0000000000 --- a/src/fs/coss/CossSwapDir.h +++ /dev/null @@ -1,146 +0,0 @@ -#ifndef __COSSSWAPDIR_H__ -#define __COSSSWAPDIR_H__ - -class StoreEntry; -class CossSwapDir; -class CossMemBuf; -class DiskIOStrategy; -class DiskIOModule; -class ConfigOptionVector; -class DiskFile; - -#include "SwapDir.h" -#include "DiskIO/IORequestor.h" - -#ifndef COSS_MEMBUF_SZ -#define COSS_MEMBUF_SZ 1048576 -#endif - -/* Note that swap_filen in sio/e are actually disk offsets too! */ - -/* What we're doing in storeCossAllocate() */ -#define COSS_ALLOC_ALLOCATE 1 -#define COSS_ALLOC_REALLOC 2 - -/// \ingroup COSS -class CossSwapDir : public SwapDir, public IORequestor -{ - -public: - CossSwapDir(); - virtual void init(); - virtual void create(); - virtual void dump(StoreEntry &)const; - ~CossSwapDir(); - virtual StoreSearch *search(String const url, HttpRequest *); - virtual bool unlinkdUseful() const; - virtual void unlink (StoreEntry &); - virtual void statfs (StoreEntry &)const; - virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; - virtual int callback(); - virtual void sync(); - virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual void openLog(); - virtual void closeLog(); - virtual int writeCleanStart(); - virtual void writeCleanDone(); - virtual void logEntry(const StoreEntry & e, int op) const; - virtual void parse (int index, char *path); - virtual void reconfigure(); - virtual void swappedOut(const StoreEntry &e); - virtual uint64_t currentSize() const { return cur_size; } - virtual uint64_t currentCount() const { return n_disk_objects; } - /* internals */ - virtual off_t storeCossFilenoToDiskOffset(sfileno); - virtual sfileno storeCossDiskOffsetToFileno(off_t); - virtual CossMemBuf *storeCossFilenoToMembuf(sfileno f); - /* IORequestor routines */ - virtual void ioCompletedNotification(); - virtual void closeCompleted(); - virtual void readCompleted(const char *buf, int len, int errflag, RefCount); - virtual void writeCompleted(int errflag, size_t len, RefCount); - //private: - int swaplog_fd; - int count; - dlink_list membufs; - - CossMemBuf *current_membuf; - off_t current_offset; /* in Blocks */ - int numcollisions; - dlink_list cossindex; - unsigned int blksz_bits; - unsigned int blksz_mask; /* just 1< theFile; - char *storeCossMemPointerFromDiskOffset(off_t offset, CossMemBuf ** mb); - void storeCossMemBufUnlock(StoreIOState::Pointer); - CossMemBuf *createMemBuf(off_t start, sfileno curfn, int *collision); - sfileno allocate(const StoreEntry * e, int which); - void startMembuf(); - StoreEntry *addDiskRestore(const cache_key *const key, - int file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t flags, - int clean); - -private: - void changeIO(DiskIOModule *module); - bool optionIOParse(char const *option, const char *value, int reconfiguring); - void optionIODump(StoreEntry * e) const; - void optionBlockSizeDump(StoreEntry *) const; - bool optionBlockSizeParse(const char *, const char *, int); - char const *stripePath() const; - ConfigOption * getOptionTree() const; - const char *ioModule; - mutable ConfigOptionVector *currentIOOptions; - const char *stripe_path; - uint64_t cur_size; ///< currently used space in the storage area - uint64_t n_disk_objects; ///< total number of objects stored -}; - -/// \ingroup COSS -void storeCossAdd(CossSwapDir *, StoreEntry *); -/// \ingroup COSS -void storeCossRemove(CossSwapDir *, StoreEntry *); -/// \ingroup COSS -void storeCossStartMembuf(CossSwapDir * SD); - -#include "StoreSearch.h" - -/// \ingroup COSS -class StoreSearchCoss : public StoreSearch -{ - -public: - StoreSearchCoss(RefCount sd); - StoreSearchCoss(StoreSearchCoss const &); - ~StoreSearchCoss(); - /* Iterator API - garh, wrong place */ - /* callback the client when a new StoreEntry is available - * or an error occurs - */ - virtual void next(void (callback)(void *cbdata), void *cbdata); - /* return true if a new StoreEntry is immediately available */ - virtual bool next(); - virtual bool error() const; - virtual bool isDone() const; - virtual StoreEntry *currentItem(); - -private: - RefCount sd; - void (*callback)(void *cbdata); - void *cbdata; - bool _done; - dlink_node * current; - dlink_node * next_; - - CBDATA_CLASS2(StoreSearchCoss); -}; - -#endif diff --git a/src/fs/coss/StoreFScoss.cc b/src/fs/coss/StoreFScoss.cc deleted file mode 100644 index 0146739669..0000000000 --- a/src/fs/coss/StoreFScoss.cc +++ /dev/null @@ -1,132 +0,0 @@ -/* - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Robert Collins - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - * Copyright (c) 2003, Robert Collins - */ - -#include "squid.h" -#include "StoreFileSystem.h" -#include "StoreFScoss.h" -#include "mgr/Registration.h" -#include "Store.h" -#include "CossSwapDir.h" -#include "store_coss.h" - -StoreFScoss StoreFScoss::_instance; - -StoreFScoss & -StoreFScoss::GetInstance() -{ - return _instance; -} - -StoreFScoss::StoreFScoss() -{ - FsAdd(*this); - registerWithCacheManager(); -} - -char const * -StoreFScoss::type() const -{ - return "coss"; -} - -void -StoreFScoss::done() -{ - /* delete coss_index_pool;coss_index_pool = NULL; XXX Should be here? */ - initialised = false; -} - -SwapDir * -StoreFScoss::createSwapDir() -{ - SwapDir *result = new CossSwapDir; - return result; -} - -void -StoreFScoss::setup() -{ - assert(!initialised); - - coss_index_pool = memPoolCreate("COSS index data", sizeof(CossIndexNode)); - initialised = true; -} - -void -StoreFScoss::registerWithCacheManager() -{ - Mgr::RegisterAction("coss", "COSS Stats", Stats, 0, 1); -} - -void -StoreFScoss::Stats(StoreEntry * sentry) -{ - GetInstance().stat(sentry); -} - -void -StoreFScoss::stat(StoreEntry *sentry) -{ - stats.stat(sentry); -} - -void -CossStats::stat(StoreEntry *sentry) -{ - const char *tbl_fmt = "%10s %10d %10d %10d\n"; - storeAppendPrintf(sentry, "\n OPS SUCCESS FAIL\n"); - storeAppendPrintf(sentry, tbl_fmt, - "open", open.ops, open.success, open.fail); - storeAppendPrintf(sentry, tbl_fmt, - "create", create.ops, create.success, create.fail); - storeAppendPrintf(sentry, tbl_fmt, - "close", close.ops, close.success, close.fail); - storeAppendPrintf(sentry, tbl_fmt, - "unlink", unlink.ops, unlink.success, unlink.fail); - storeAppendPrintf(sentry, tbl_fmt, - "read", read.ops, read.success, read.fail); - storeAppendPrintf(sentry, tbl_fmt, - "write", write.ops, write.success, write.fail); - storeAppendPrintf(sentry, tbl_fmt, - "s_write", stripe_write.ops, stripe_write.success, stripe_write.fail); - - storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "stripes: %d\n", stripes); - storeAppendPrintf(sentry, "alloc.alloc: %d\n", alloc.alloc); - storeAppendPrintf(sentry, "alloc.realloc: %d\n", alloc.realloc); - storeAppendPrintf(sentry, "alloc.collisions: %d\n", alloc.collisions); - storeAppendPrintf(sentry, "disk_overflows: %d\n", disk_overflows); - storeAppendPrintf(sentry, "stripe_overflows: %d\n", stripe_overflows); - storeAppendPrintf(sentry, "open_mem_hits: %d\n", open_mem_hits); - storeAppendPrintf(sentry, "open_mem_misses: %d\n", open_mem_misses); -} diff --git a/src/fs/coss/StoreFScoss.h b/src/fs/coss/StoreFScoss.h deleted file mode 100644 index 81ebf23a6f..0000000000 --- a/src/fs/coss/StoreFScoss.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - * Copyright (c) 2003, Robert Collins - */ -#ifndef SQUID_STOREFSCOSS_H -#define SQUID_STOREFSCOSS_H - -class StoreEntry; - -/** - \defgroup COSS COSS Storage Filesystem - \ingroup FileSystems - */ - -/// \ingroup COSS -class CossStats -{ - -public: - void stat(StoreEntry * sentry); - int stripes; - - struct { - int alloc; - int realloc; - int collisions; - } alloc; - int disk_overflows; - int stripe_overflows; - int open_mem_hits; - int open_mem_misses; - - struct { - int ops; - int success; - int fail; - } - - open, create, close, unlink, read, write, stripe_write; -}; - -class CacheManager; - -#include "StoreFileSystem.h" - -/// \ingroup COSS, FileSystems -class StoreFScoss : public StoreFileSystem -{ - -public: - static StoreFScoss &GetInstance(); - static void Stats(StoreEntry * sentry); - StoreFScoss(); - virtual ~StoreFScoss() {} - - virtual char const *type() const; - virtual SwapDir *createSwapDir(); - virtual void done(); - virtual void registerWithCacheManager(void); - virtual void setup(); - /* Not implemented */ - StoreFScoss (StoreFScoss const &); - StoreFScoss &operator=(StoreFScoss const &); - void stat(StoreEntry * sentry); - CossStats stats; - -private: - static StoreFScoss _instance; -}; - -#endif /* SQUID_STOREFSCOSS_H */ diff --git a/src/fs/coss/coss-notes.txt b/src/fs/coss/coss-notes.txt deleted file mode 100644 index 953d492677..0000000000 --- a/src/fs/coss/coss-notes.txt +++ /dev/null @@ -1,128 +0,0 @@ -COSS notes - -Amos Jeffries - -COSS Support is not stable yet in Squid-3. Please do not use. - -Any help porting the stability fixes from Squid 2.6 and 2.7 -is very welcome though to resolve this issue. - - -Adrian Chadd - -COSS is a Cyclic Object storage system originally designed by -Eric Stern . The idea has been extended -and worked into the current framework by myself. - -In these notes I'll discuss the current implementation of COSS -and note where the implementation differed from Eric's original -idea and why the design changes were made. - - -COSS basics ------------ - -COSS works with a single file. Eventually the file may actually be -a raw disk device, but since squid doesn't cache the disk reads -in memory the OS buffer cache will need to be employed for reasonable -performance. For the purposes of this discussion the COSS storage -device will be referred to as a file. - -Each stripe is a fixed size an in a fixed position in the file. The -stripe size is a compile-time option. - -As objects are written to a COSS stripe, their place is pre-reserved -and data is copied into a memory copy of the stripe. Because of this, -the object size must be known before it can be stored in a COSS -filesystem. (Hence the max-size requirement with a coss cache_dir.) - -When a stripe is filled, the stripe is written to disk, and a new -memory stripe is created. - -When objects are read back from the COSS file, they can either come -from a stripe in-memory (the current one, or one being written), -or from the disk. If the object is still in a memory stripe, then -it is copied from memory rather than read of disk. - -If an object is read from disk, it is re-written to the head of -the current stripe (just as if it were a new object.) This is required -for correct operation of the replacement policy, detailed below. - -When the entire COSS file is full, the current stripe again becomes the -fist stripe in the file, and the objects in that stripe are released. -Since the objects on disk are kept in a strict LRU representing the -replacement policy LRU linking the StoreEntry's together, this simply -involves walking the tail of the LRU and freeing entries until we -hit an entry in the next stripe. - - -COSS implementation details ---------------------------- - -* The stripe size is fixed. In the original COSS code, Eric optimised - this a little by allowing the stripes to be truncated to not - waste disk space at the end of the stripe. This was removed - to simplify the allocation code slightly and make things easier - when the store log and checksums are combined in the stripe - for faster rebuilds. - -* COSS currently copies object memory around WAY too much. This needs - to be fixed eventually. - -* It would be nice if the storeRead() interface were a little smarter - and allowed the filesystem to return as much of an object as possible. - This would be good for COSS since the read from disk could be simplified - to use a single OS read() call - this would work really well for - the object types COSS is designed to cache. - -* The original coss code used file_read() and file_write() for disk IO. - The file_* routines were initially used to implement async disk IO, - and Eric probably wrote some async disk code for windows. - I've written a very very simple async_io.c module which uses POSIX - AIO to implement the async IO. POSIX AIO is well-suited to the - disk IO COSS performs. - -COSS direction --------------- - -Eventually, when more of squid is rewritten, I'm going to replace -the replacement policy with something a little more flexible. -A shortcut would be to use a slab allocator and have one slab per -stripe for the StoreEntry's. When it comes time to replace a stripe, -you can just treat the stripe as an array. This would not work -well in the current squid codebase, but it would work well in the -planned rewrite. This would also allow alternate replacement policies -to be used. Oh, it'd cut down the storage requirements per -StoreEntry by two pointers (8 bytes on the i386.) - -Notes by DW July 23, 2003 -------------------------- - -Fixed up swap_filen -> offset implementation. Now user can use a -block-size setting to determine the maximum COSS cache_dir size. - -Fixed bug when cached response is larger than COSS stripe size. -Now require max-size to be less than COSS_MEMBUF_SZ. - -Fixed a lockcount bug. Some aborted requests for cache hits failed -to unlock the CossMemBuf because storeCossReadDone isn't called again. -Solution is to add locked_membuf pointer to CossState structure and -always unlock it if set. This is probably more reliable than -unlocking based on diskstart/diskend offsets. - -I'm worried that COSS is susceptible to a denial-of-service. If -the user can create N cache misses for responses about as large as -COSS_MEMBUF_SZ, then COSS probably allocates N membufs (stripes) -at the same time. For large enough values of N, this should cause -a malloc failure. Solution may be to refuse to allocate new stripes -(thus returning failure for cache misses and hits) after so many -have already been allocated. - -Adrian's code has this comment: - - /* Since we're not supporting NOTIFY anymore, lets fail */ - assert(which != COSS_ALLOC_NOTIFY); - -However, COSS_ALLOC_NOTIFY was still present in the store_dir_coss.c -rebuild routines. To avoid assertions during rebuild, I commented -out the storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY) call. diff --git a/src/fs/coss/store_coss.h b/src/fs/coss/store_coss.h deleted file mode 100644 index 482cdf9140..0000000000 --- a/src/fs/coss/store_coss.h +++ /dev/null @@ -1,133 +0,0 @@ -#ifndef __COSS_H__ -#define __COSS_H__ - -#include "SwapDir.h" - -#ifndef COSS_MEMBUF_SZ -#define COSS_MEMBUF_SZ 1048576 -#endif - -/** \note swap_filen in sio/e are actually disk offsets too! */ - -/** What we're doing in storeCossAllocate() */ -#define COSS_ALLOC_NOTIFY 0 - -/** What we're doing in storeCossAllocate() */ -#define COSS_ALLOC_ALLOCATE 1 - -/** What we're doing in storeCossAllocate() */ -#define COSS_ALLOC_REALLOC 2 - -class CossSwapDir; - -/// \ingroup COSS -class CossMemBuf -{ - -public: - void describe(int level, int line); - void maybeWrite(CossSwapDir * SD); - void write(CossSwapDir * SD); - dlink_node node; - off_t diskstart; /* in blocks */ - off_t diskend; /* in blocks */ - CossSwapDir *SD; - int lockcount; - char buffer[COSS_MEMBUF_SZ]; - - struct _cossmembuf_flags { - _cossmembuf_flags() : full(false), writing(false) {} - bool full; - bool writing; - } flags; -}; - -/// \ingroup COSS -struct _cossindex { - /** - \note The dlink_node MUST be the first member of the structure. - * This member is later pointer typecasted to coss_index_node *. - */ - dlink_node node; -}; - -/** - \ingroup COSS - * Per-storeiostate info - */ -class CossState : public StoreIOState -{ - -public: - MEMPROXY_CLASS(CossState); - CossState(CossSwapDir *); - ~CossState(); - - char *readbuffer; - char *requestbuf; - size_t requestlen; - size_t requestoffset; /* in blocks */ - int64_t reqdiskoffset; /* in blocks */ - - struct CossFlags { - bool reading; - bool writing; - } flags; - - CossMemBuf *locked_membuf; - off_t st_size; - void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); - void write(char const *buf, size_t size, off_t offset, FREE * free_func); - virtual void close(int); - void doCallback(int errflag); - void lockMemBuf(); - - CossSwapDir *SD; -}; - -MEMPROXY_CLASS_INLINE(CossState); - -/// \ingroup COSS -typedef struct _cossindex CossIndexNode; - -/** - \ingroup COSS - * Whether the coss system has been setup or not - */ -extern int coss_initialised; -/// \ingroup COSS -extern MemAllocator *coss_membuf_pool; -/// \ingroup COSS -extern MemAllocator *coss_index_pool; - -#include "DiskIO/ReadRequest.h" - -/// \ingroup COSS -class CossRead : public ReadRequest -{ - -public: - CossRead(ReadRequest const &base, StoreIOState::Pointer anSio) : ReadRequest(base) , sio(anSio) {} - - StoreIOState::Pointer sio; - -private: - CBDATA_CLASS2(CossRead); -}; - -#include "DiskIO/WriteRequest.h" - -/// \ingroup COSS -class CossWrite : public WriteRequest -{ - -public: - CossWrite(WriteRequest const &base, CossMemBuf *aBuf) : WriteRequest(base) , membuf(aBuf) {} - - CossMemBuf *membuf; - -private: - CBDATA_CLASS2(CossWrite); -}; - -#endif diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc deleted file mode 100644 index 72a592ced3..0000000000 --- a/src/fs/coss/store_dir_coss.cc +++ /dev/null @@ -1,1203 +0,0 @@ -/* - * DEBUG: section 47 Store COSS Directory Routines - * AUTHOR: Eric Stern - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "CossSwapDir.h" -#include "cache_cf.h" -#include "Store.h" -#include "store_coss.h" -#include "disk.h" -#include "event.h" -#include "fde.h" -#include "SwapDir.h" -#include "StoreSwapLogData.h" -#include "DiskIO/DiskIOModule.h" -#include "DiskIO/DiskIOStrategy.h" -#include "DiskIO/ReadRequest.h" -#include "ConfigOption.h" -#include "StoreFScoss.h" -#include "Parsing.h" -#include "store_key_md5.h" -#include "swap_log_op.h" -#include "store_rebuild.h" -#include "SquidConfig.h" -#include "SquidMath.h" - -#if HAVE_ERRNO_H -#include -#endif - -#define STORE_META_BUFSZ 4096 - -int n_coss_dirs = 0; -/* static int last_coss_pick_index = -1; */ -MemAllocator *coss_index_pool = NULL; - -typedef struct _RebuildState RebuildState; - -struct _RebuildState { - CossSwapDir *sd; - int n_read; - FILE *log; - int speed; - - struct { - unsigned int clean:1; - } flags; - - StoreRebuildData counts; -}; - -static char *storeCossDirSwapLogFile(SwapDir *, const char *); -static EVH storeCossRebuildFromSwapLog; -static void storeCossDirRebuild(CossSwapDir * sd); -static void storeCossDirCloseTmpSwapLog(CossSwapDir * sd); -static FILE *storeCossDirOpenTmpSwapLog(CossSwapDir *, int *, int *); - -static char * -storeCossDirSwapLogFile(SwapDir * sd, const char *ext) -{ - LOCAL_ARRAY(char, path, MAXPATHLEN); - LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); - LOCAL_ARRAY(char, digit, 32); - char *pathtmp2; - - if (Config.Log.swap) { - xstrncpy(pathtmp, sd->path, MAXPATHLEN - 64); - pathtmp2 = pathtmp; - - while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) - *pathtmp2 = '.'; - - while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') - pathtmp[strlen(pathtmp) - 1] = '\0'; - - for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2); - snprintf(path, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); - - if (strncmp(path, Config.Log.swap, MAXPATHLEN - 64) == 0) { - strcat(path, "."); - snprintf(digit, 32, "%02d", sd->index); - strncat(path, digit, 3); - } - } else { - xstrncpy(path, sd->path, MAXPATHLEN - 64); - strcat(path, "/swap.state"); - } - - if (ext) - strncat(path, ext, 16); - - return path; -} - -void -CossSwapDir::openLog() -{ - char *logPath; - logPath = storeCossDirSwapLogFile(this, NULL); - swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); - - if (swaplog_fd < 0) { - debugs(47, DBG_IMPORTANT, "" << logPath << ": " << xstrerror()); - fatal("storeCossDirOpenSwapLog: Failed to open swap log."); - } - - debugs(47, 3, "Cache COSS Dir #" << index << " log opened on FD " << swaplog_fd); -} - -void -CossSwapDir::closeLog() -{ - if (swaplog_fd < 0) /* not open */ - return; - - file_close(swaplog_fd); - - debugs(47, 3, "Cache COSS Dir #" << index << " log closed on FD " << swaplog_fd); - - swaplog_fd = -1; -} - -void -CossSwapDir::ioCompletedNotification() -{ - if (theFile->error()) { - debugs(47, DBG_IMPORTANT, "" << path << ": " << xstrerror()); - fatal("storeCossDirInit: Failed to open a COSS file."); - } -} - -void -CossSwapDir::closeCompleted() -{ - theFile = NULL; -} - -void -CossSwapDir::readCompleted(const char *buf, int len, int errflag, RefCount aRequest) -{ - CossRead* cossRead= dynamic_cast(aRequest.getRaw()); - assert (cossRead); - StoreIOState::Pointer sio = cossRead->sio; - void *cbdata; - StoreIOState::STRCB *callback = sio->read.callback; - char *p; - CossState *cstate = dynamic_cast(sio.getRaw()); - ssize_t rlen; - - debugs(79, 3, "storeCossReadDone: fileno " << sio->swap_filen << ", len " << len); - cstate->flags.reading = false; - - if (errflag) { - ++ StoreFScoss::GetInstance().stats.read.fail; - - if (errflag > 0) { - errno = errflag; - debugs(79, DBG_IMPORTANT, "storeCossReadDone: error: " << xstrerror()); - } else { - debugs(79, DBG_IMPORTANT, "storeCossReadDone: got failure (" << errflag << ")"); - } - - rlen = -1; - } else { - ++ StoreFScoss::GetInstance().stats.read.success; - - if (cstate->readbuffer == NULL) { - cstate->readbuffer = (char *)xmalloc(cstate->st_size); - p = storeCossMemPointerFromDiskOffset(storeCossFilenoToDiskOffset(sio->swap_filen), - NULL); - memcpy(cstate->readbuffer, p, cstate->st_size); - } - - sio->offset_ += len; - memcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset], - cstate->requestlen); - rlen = (size_t) cstate->requestlen; - } - - assert(callback); - sio->read.callback = NULL; - - if (cbdataReferenceValidDone(sio->read.callback_data, &cbdata)) - callback(cbdata, cstate->requestbuf, rlen, sio); -} - -void -CossSwapDir::writeCompleted(int errflag, size_t len, RefCount writeRequest) -{ - CossWrite* cossWrite= dynamic_cast(writeRequest.getRaw()); - assert (cossWrite); - - debugs(79, 3, "storeCossWriteMemBufDone: buf " << cossWrite->membuf << ", len " << len); - - if (errflag) { - ++ StoreFScoss::GetInstance().stats.stripe_write.fail; - debugs(79, DBG_IMPORTANT, "storeCossWriteMemBufDone: got failure (" << errflag << ")"); - debugs(79, DBG_IMPORTANT, "size=" << cossWrite->membuf->diskend - cossWrite->membuf->diskstart); - } else { - ++ StoreFScoss::GetInstance().stats.stripe_write.success; - } - - dlinkDelete(&cossWrite->membuf->node, &membufs); - cbdataFree(cossWrite->membuf); - -- StoreFScoss::GetInstance().stats.stripes; -} - -void -CossSwapDir::changeIO(DiskIOModule *module) -{ - DiskIOStrategy *anIO = module->createStrategy(); - safe_free(ioModule); - ioModule = xstrdup(module->type()); - - delete io; - io = anIO; - /* Change the IO Options */ - - if (currentIOOptions == NULL) - currentIOOptions = new ConfigOptionVector(); - - if (currentIOOptions->options.size() > 3) - delete currentIOOptions->options.pop_back(); - - /* TODO: factor out these 4 lines */ - ConfigOption *ioOptions = NULL; - - if (io) - ioOptions = io->getOptionTree(); - - if (ioOptions) - currentIOOptions->options.push_back(ioOptions); -} - -bool -CossSwapDir::optionIOParse(char const *option, const char *value, int reconfiguring) -{ - if (strcmp(option, "IOEngine") != 0) - return false; - - if (reconfiguring) - /* silently ignore this */ - return true; - - if (!value) - self_destruct(); - - DiskIOModule *module = DiskIOModule::Find(value); - - if (!module) - self_destruct(); - - changeIO(module); - - return true; -} - -void -CossSwapDir::optionIODump(StoreEntry * e) const -{ - storeAppendPrintf(e, " IOEngine=%s", ioModule); -} - -ConfigOption * -CossSwapDir::getOptionTree() const -{ - ConfigOption *parentResult = SwapDir::getOptionTree(); - - if (currentIOOptions == NULL) - currentIOOptions = new ConfigOptionVector(); - - currentIOOptions->options.push_back(parentResult); - - currentIOOptions->options.push_back(new ConfigOptionAdapter(*const_cast(this), &CossSwapDir::optionIOParse, &CossSwapDir::optionIODump)); - - currentIOOptions->options.push_back( - new ConfigOptionAdapter(*const_cast(this), - &CossSwapDir::optionBlockSizeParse, - &CossSwapDir::optionBlockSizeDump)); - - ConfigOption *ioOptions = NULL; - - if (io) - ioOptions = io->getOptionTree(); - - if (ioOptions) - currentIOOptions->options.push_back(ioOptions); - - ConfigOption* result = currentIOOptions; - - currentIOOptions = NULL; - - return result; -} - -void -CossSwapDir::init() -{ - /* FIXME: SwapDirs aren't refcounted. We call IORequestor calls, which - * are refcounted. SO, we up our count once to avoid implicit delete's. - */ - RefCountReference(); - io->init(); - openLog(); - storeCossDirRebuild(this); - theFile = io->newFile(stripePath()); - theFile->open(O_RDWR | O_CREAT, 0644, this); - - ++n_coss_dirs; - /* - * fs.blksize is normally determined by calling statvfs() etc, - * but we just set it here. It is used in accounting the - * total store size, and is reported in cachemgr 'storedir' - * page. - */ - fs.blksize = 1 << blksz_bits; -} - -void -storeCossRemove(CossSwapDir * sd, StoreEntry * e) -{ - CossIndexNode *coss_node = (CossIndexNode *)e->repl.data; - e->repl.data = NULL; - dlinkDelete(&coss_node->node, &sd->cossindex); - coss_index_pool->freeOne(coss_node); - sd->count -= 1; -} - -void -storeCossAdd(CossSwapDir * sd, StoreEntry * e) -{ - CossIndexNode *coss_node = (CossIndexNode *)coss_index_pool->alloc(); - assert(!e->repl.data); - e->repl.data = coss_node; - dlinkAdd(e, &coss_node->node, &sd->cossindex); - sd->count += 1; -} - -static void -storeCossRebuildComplete(void *data) -{ - RebuildState *rb = (RebuildState *)data; - CossSwapDir *sd = rb->sd; - sd->startMembuf(); - -- StoreController::store_dirs_rebuilding; - storeCossDirCloseTmpSwapLog(rb->sd); - storeRebuildComplete(&rb->counts); - cbdataFree(rb); -} - -static void -storeCossRebuildFromSwapLog(void *data) -{ - RebuildState *rb = (RebuildState *)data; - StoreEntry *e = NULL; - StoreSwapLogData s; - size_t ss = sizeof(StoreSwapLogData); - double x; - assert(rb != NULL); - /* load a number of objects per invocation */ - - for (int aCount = 0; aCount < rb->speed; ++aCount) { - if (fread(&s, ss, 1, rb->log) != 1) { - debugs(47, DBG_IMPORTANT, "Done reading " << rb->sd->path << " swaplog (" << rb->n_read << " entries)"); - fclose(rb->log); - rb->log = NULL; - storeCossRebuildComplete(rb); - return; - } - - ++ rb->n_read; - - if (s.op <= SWAP_LOG_NOP) - continue; - - if (s.op >= SWAP_LOG_MAX) - continue; - - debugs(47, 3, "storeCossRebuildFromSwapLog: " << - swap_log_op_str[(int) s.op] << " " << storeKeyText(s.key) << - " "<< std::setfill('0') << std::hex << std::uppercase << - std::setw(8) << s.swap_filen); - - if (s.op == SWAP_LOG_ADD) { - (void) 0; - } else if (s.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy */ - - if ((e = rb->sd->get - (s.key)) != NULL && s.lastref > e->lastref) { - /* - * Make sure we don't unlink the file, it might be - * in use by a subsequent entry. Also note that - * we don't have to subtract from cur_size because - * adding to cur_size happens in the cleanup procedure. - */ - e->expireNow(); - e->releaseRequest(); - - if (e->swap_filen > -1) { - e->swap_filen = -1; - } - - e->release(); - /* Fake an unlink here, this is a bad hack :( */ - storeCossRemove(rb->sd, e); - -- rb->counts.objcount; - ++ rb->counts.cancelcount; - } - continue; - } else { - x = log(static_cast(++rb->counts.bad_log_op)) / log(10.0); - - if (0.0 == x - (double) - (int) x) - debugs(47, DBG_IMPORTANT, "WARNING: " << rb->counts.bad_log_op << " invalid swap log entries found"); - - ++ rb->counts.invalid; - - continue; - } - - if ((++rb->counts.scancount & 0xFFF) == 0) { - - struct stat sb; - - if (0 == fstat(fileno(rb->log), &sb)) - storeRebuildProgress(rb->sd->index, - (int) sb.st_size / ss, rb->n_read); - } - - if (EBIT_TEST(s.flags, KEY_PRIVATE)) { - ++ rb->counts.badflags; - continue; - } - - e = rb->sd->get - (s.key); - - if (e) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - ++ rb->counts.dupcount; - continue; - } - - ++ rb->counts.objcount; - - e = rb->sd->addDiskRestore(s.key, - s.swap_filen, - s.swap_file_sz, - s.expires, - s.timestamp, - s.lastref, - s.lastmod, - s.refcount, - s.flags, - (int) rb->flags.clean); - - storeDirSwapLog(e, SWAP_LOG_ADD); - } - - eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); -} - -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -StoreEntry * -CossSwapDir::addDiskRestore(const cache_key *const key, - int file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t flags, - int clean) -{ - StoreEntry *e = NULL; - debugs(47, 5, "storeCossAddDiskRestore: " << storeKeyText(key) << - ", fileno="<< std::setfill('0') << std::hex << std::uppercase << - std::setw(8) << file_number); - - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new StoreEntry(); - e->store_status = STORE_OK; - e->swap_dirn = index; - e->setMemStatus(NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_filen = file_number; - e->swap_file_sz = swap_file_sz; - e->lock_count = 0; - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flags = flags; - EBIT_SET(e->flags, ENTRY_CACHABLE); - EBIT_CLR(e->flags, RELEASE_REQUEST); - EBIT_CLR(e->flags, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flags, ENTRY_VALIDATED); - cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); - ++n_disk_objects; - e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ - storeCossAdd(this, e); - assert(e->swap_filen >= 0); - return e; -} - -CBDATA_TYPE(RebuildState); -static void -storeCossDirRebuild(CossSwapDir * sd) -{ - RebuildState *rb; - int clean = 0; - int zero = 0; - FILE *fp; - CBDATA_INIT_TYPE(RebuildState); - rb = cbdataAlloc(RebuildState); - rb->sd = sd; - rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; - rb->flags.clean = (unsigned int) clean; - /* - * If the swap.state file exists in the cache_dir, then - * we'll use storeCossRebuildFromSwapLog(). - */ - fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero); - debugs(47, DBG_IMPORTANT, "Rebuilding COSS storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")"); - rb->log = fp; - ++ StoreController::store_dirs_rebuilding; - - if (!clean || fp == NULL) { - /* COSS cannot yet rebuild from a dirty state. If the log - * is dirty then the COSS contents is thrown away. - * Why? I guess it is because some contents will be lost, - * and COSS cannot verify this.. - */ - - if (fp != NULL) - fclose(fp); - - /* - * XXX Make sure we don't trigger an assertion if this is the first - * storedir, since if we are, this call will cause storeRebuildComplete - * to prematurely complete the rebuild process, and then some other - * storedir will try to rebuild and eventually die. - */ - eventAdd("storeCossRebuildComplete", storeCossRebuildComplete, rb, 0.0, 0); - - return; - } - - eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); -} - -static void -storeCossDirCloseTmpSwapLog(CossSwapDir * sd) -{ - char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); - char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); - int anfd; - file_close(sd->swaplog_fd); - - if (xrename(new_path, swaplog_path) < 0) { - fatal("storeCossDirCloseTmpSwapLog: rename failed"); - } - - anfd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); - - if (anfd < 0) { - debugs(50, DBG_IMPORTANT, "" << swaplog_path << ": " << xstrerror()); - fatal("storeCossDirCloseTmpSwapLog: Failed to open swap log."); - } - - safe_free(swaplog_path); - safe_free(new_path); - sd->swaplog_fd = anfd; - debugs(47, 3, "Cache COSS Dir #" << sd->index << " log opened on FD " << anfd); -} - -static FILE * -storeCossDirOpenTmpSwapLog(CossSwapDir * sd, int *clean_flag, int *zero_flag) -{ - char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); - char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); - char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); - - struct stat log_sb; - - struct stat clean_sb; - FILE *fp; - int anfd; - - if (::stat(swaplog_path, &log_sb) < 0) { - debugs(50, DBG_IMPORTANT, "Cache COSS Dir #" << sd->index << ": No log file"); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return NULL; - } - - *zero_flag = log_sb.st_size == 0 ? 1 : 0; - /* close the existing write-only FD */ - - if (sd->swaplog_fd >= 0) - file_close(sd->swaplog_fd); - - /* open a write-only FD for the new log */ - anfd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - - if (anfd < 0) { - debugs(50, DBG_IMPORTANT, "" << new_path << ": " << xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); - } - - sd->swaplog_fd = anfd; - /* open a read-only stream of the old log */ - fp = fopen(swaplog_path, "rb"); - - if (fp == NULL) { - debugs(50, DBG_CRITICAL, "" << swaplog_path << ": " << xstrerror()); - fatal("Failed to open swap log for reading"); - } - - memset(&clean_sb, '\0', sizeof(struct stat)); - - if (::stat(clean_path, &clean_sb) < 0) - *clean_flag = 0; - else if (clean_sb.st_mtime < log_sb.st_mtime) - *clean_flag = 0; - else - *clean_flag = 1; - - safeunlink(clean_path, 1); - - safe_free(swaplog_path); - - safe_free(clean_path); - - safe_free(new_path); - - return fp; -} - -class CossCleanLog : public SwapDir::CleanLog -{ - -public: - CossCleanLog(CossSwapDir *); - virtual const StoreEntry *nextEntry(); - virtual void write(StoreEntry const &); - char *cur; - char *newLog; - char *cln; - char *outbuf; - off_t outbuf_offset; - int fd; - dlink_node *current; - CossSwapDir *sd; -}; - -#define CLEAN_BUF_SZ 16384 - -CossCleanLog::CossCleanLog(CossSwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), - outbuf_offset(0), fd(-1),current(NULL), sd(aSwapDir) -{} - -/* - * Begin the process to write clean cache state. For COSS this means - * opening some log files and allocating write buffers. Return 0 if - * we succeed, and assign the 'func' and 'data' return pointers. - */ -int -CossSwapDir::writeCleanStart() -{ - CossCleanLog *state = new CossCleanLog(this); -#if HAVE_FCHMOD - - struct stat sb; -#endif - - state->newLog = xstrdup(storeCossDirSwapLogFile(this, ".clean")); - state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - cleanLog = NULL; - - if (state->fd < 0) { - xfree(state->newLog); - delete state; - return -1; - } - - state->cur = xstrdup(storeCossDirSwapLogFile(this, NULL)); - state->cln = xstrdup(storeCossDirSwapLogFile(this, ".last-clean")); - state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); - state->outbuf_offset = 0; - ::unlink(state->cln); - state->current = cossindex.tail; - debugs(50, 3, "storeCOssDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); -#if HAVE_FCHMOD - - if (::stat(state->cur, &sb) == 0) - fchmod(state->fd, sb.st_mode); - -#endif - - cleanLog = state; - - return 0; -} - -/* RBC 20050101 - I think there is a race condition here, - * *current can be freed as its not ref counted, if/when - * the store overruns the log writer - */ -const StoreEntry * -CossCleanLog::nextEntry() -{ - const StoreEntry *entry; - - if (!current) - return NULL; - - entry = (const StoreEntry *) current->data; - - current = current->prev; - - return entry; -} - -/* - * "write" an entry to the clean log file. - */ -void -CossCleanLog::write(StoreEntry const &e) -{ - CossCleanLog *state = this; - StoreSwapLogData s; - static size_t ss = sizeof(StoreSwapLogData); - s.op = (char) SWAP_LOG_ADD; - s.swap_filen = e.swap_filen; - s.timestamp = e.timestamp; - s.lastref = e.lastref; - s.expires = e.expires; - s.lastmod = e.lastmod; - s.swap_file_sz = e.swap_file_sz; - s.refcount = e.refcount; - s.flags = e.flags; - memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); - memcpy(outbuf + outbuf_offset, &s, ss); - outbuf_offset += ss; - /* buffered write */ - - if (outbuf_offset + ss > CLEAN_BUF_SZ) { - if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { - debugs(50, DBG_CRITICAL, "storeCossDirWriteCleanLogs: " << newLog << ": write: " << xstrerror()); - debugs(50, DBG_CRITICAL, "storeCossDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(fd); - fd = -1; - unlink(newLog); - sd->cleanLog = NULL; - delete state; - return; - } - - outbuf_offset = 0; - } -} - -void -CossSwapDir::writeCleanDone() -{ - CossCleanLog *state = (CossCleanLog *)cleanLog; - - if (NULL == state) - return; - - if (state->fd < 0) - return; - - if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debugs(50, DBG_CRITICAL, "storeCossDirWriteCleanLogs: " << state->newLog << ": write: " << xstrerror()); - debugs(50, DBG_CRITICAL, "storeCossDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(state->fd); - state->fd = -1; - ::unlink(state->newLog); - } - - safe_free(state->outbuf); - /* - * You can't rename open files on Microsoft "operating systems" - * so we have to close before renaming. - */ - closeLog(); - /* save the fd value for a later test */ - int anfd = state->fd; - /* rename */ - - if (state->fd >= 0) { -#if _SQUID_OS2_ || _SQUID_WINDOWS_ - file_close(state->fd); - state->fd = -1; -#endif - - xrename(state->newLog, state->cur); - } - - /* touch a timestamp file if we're not still validating */ - if (StoreController::store_dirs_rebuilding) - (void) 0; - else if (anfd < 0) - (void) 0; - else - file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); - - /* close */ - safe_free(state->cur); - - safe_free(state->newLog); - - safe_free(state->cln); - - if (state->fd >= 0) - file_close(state->fd); - - state->fd = -1; - - delete state; - - cleanLog = NULL; -} - -static void -FreeObject(void *address) -{ - StoreSwapLogData *anObject = static_cast (address); - delete anObject; -} - -void -CossSwapDir::logEntry(const StoreEntry & e, int op) const -{ - StoreSwapLogData *s = new StoreSwapLogData; - s->op = (char) op; - s->swap_filen = e.swap_filen; - s->timestamp = e.timestamp; - s->lastref = e.lastref; - s->expires = e.expires; - s->lastmod = e.lastmod; - s->swap_file_sz = e.swap_file_sz; - s->refcount = e.refcount; - s->flags = e.flags; - memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); - file_write(swaplog_fd, - -1, - s, - sizeof(StoreSwapLogData), - NULL, - NULL, - &FreeObject); -} - -void -CossSwapDir::create() -{ - debugs (47, 3, "Creating swap space in " << path); - - struct stat swap_sb; - int swap; - - if (::stat(path, &swap_sb) < 0) { - debugs (47, 2, "COSS swap space space being allocated."); - mkdir(path, 0700); - } - - /* should check here for directories instead of files, and for file size - * TODO - if nothing changes, there is nothing to do - */ - swap = open(stripePath(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0600); - - /* TODO just set the file size */ - char block[1024]; - Must(maxSize() % sizeof(block) == 0); - memset(block, '\0', sizeof(block)); - - for (uint64_t offset = 0; offset < maxSize(); offset += sizeof(block)) { - if (write (swap, block, sizeof(block)) != sizeof(block)) { - debugs (47, 0, "Failed to create COSS swap space in " << path); - } - } - - close (swap); - -} - -/* we are shutting down, flush all membufs to disk */ -CossSwapDir::~CossSwapDir() -{ - io->sync(); - - if (theFile != NULL) - theFile->close(); - - delete io; - - closeLog(); - - --n_coss_dirs; - - safe_free(ioModule); - - safe_free(stripe_path); -} - -bool -CossSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const -{ - if (!SwapDir::canStore(e, diskSpaceNeeded, load)) - return false; - - load = io->load(); - return true; -} - -/* - * storeCossDirCallback - do the IO completions - */ -int -CossSwapDir::callback() -{ - return io->callback(); -} - -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ - -void -CossSwapDir::statfs(StoreEntry & sentry) const -{ - storeAppendPrintf(&sentry, "\n"); - storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10); - storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); - storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", - Math::doublePercent(currentSize(), maxSize()) ); - storeAppendPrintf(&sentry, "Number of object collisions: %d\n", (int) numcollisions); -#if 0 - /* is this applicable? I Hope not .. */ - storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", - SD->map->numFilesInMap(), SD->map->capacity(), - Math::intPercent(SD->map->numFilesInMap(), SD->map->capacity())); -#endif - - // storeAppendPrintf(&sentry, "Pending operations: %d out of %d\n", io->aq.aq_numpending, MAX_ASYNCOP); - storeAppendPrintf(&sentry, "Flags:"); - - if (flags.selected) - storeAppendPrintf(&sentry, " SELECTED"); - - if (flags.read_only) - storeAppendPrintf(&sentry, " READ-ONLY"); - - storeAppendPrintf(&sentry, "\n"); -} - -void -CossSwapDir::parse(int anIndex, char *aPath) -{ - const int i = GetInteger(); - if (i <= 0) - fatal("storeCossDirParse: invalid size value"); - - index = anIndex; - - path = xstrdup(aPath); - - max_size = static_cast(i) << 20; // MBytes to Bytes - - parseOptions(0); - - if (NULL == io) - changeIO(DiskIOModule::FindDefault()); - - /* Enforce maxobjsize being set to something */ - if (max_objsize == -1) - fatal("COSS requires max-size to be set to something other than -1!\n"); - - if (max_objsize > COSS_MEMBUF_SZ) - fatalf("COSS max-size option must be less than COSS_MEMBUF_SZ (%d)\n", - COSS_MEMBUF_SZ); - - // check that we won't overflow sfileno later. - const uint64_t max_offset = (uint64_t)SwapFilenMax << blksz_bits; - - if (maxSize() > max_offset) { - debugs(47, DBG_CRITICAL, "COSS block-size = " << (1<(i) << 20; // MBytes to Bytes - - if (size == maxSize()) - debugs(3, DBG_IMPORTANT, "Cache COSS dir '" << path << "' size remains unchanged at " << i << " MB"); - else { - debugs(3, DBG_IMPORTANT, "Cache COSS dir '" << path << "' size changed to " << i << " MB"); - max_size = size; - } - - /* Enforce maxobjsize being set to something */ - if (max_objsize == -1) - fatal("COSS requires max-size to be set to something other than -1!\n"); -} - -void -CossSwapDir::swappedOut(const StoreEntry &e) -{ - cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); - ++n_disk_objects; -} - -void -CossSwapDir::dump(StoreEntry &entry)const -{ - storeAppendPrintf(&entry, " %" PRIu64, maxSize() >> 20); - dumpOptions(&entry); -} - -CossSwapDir::CossSwapDir() : SwapDir ("coss"), swaplog_fd(-1), count(0), current_membuf (NULL), current_offset(0), numcollisions(0), blksz_bits(0), io (NULL), ioModule(NULL), currentIOOptions(new ConfigOptionVector()), stripe_path(NULL), cur_size(0), n_disk_objects(0) -{ - membufs.head = NULL; - membufs.tail = NULL; - cossindex.head = NULL; - cossindex.tail = NULL; - blksz_mask = (1 << blksz_bits) - 1; - repl = NULL; -} - -bool -CossSwapDir::optionBlockSizeParse(const char *option, const char *value, int reconfiguring) -{ - assert(option); - - if (strcmp(option, "block-size") != 0) - return false; - - if (!value) - self_destruct(); - - int blksz = atoi(value); - - if (blksz == (1 << blksz_bits)) - /* no change */ - return true; - - if (reconfiguring) { - debugs(47, DBG_CRITICAL, "WARNING: cannot change COSS block-size while Squid is running"); - return false; - } - - int nbits = 0; - int check = blksz; - - while (check > 1) { - ++nbits; - check >>= 1; - } - - check = 1 << nbits; - - if (check != blksz) - fatal("COSS block-size must be a power of 2\n"); - - if (nbits > 13) - fatal("COSS block-size must be 8192 or smaller\n"); - - blksz_bits = nbits; - - blksz_mask = (1 << blksz_bits) - 1; - - return true; -} - -void -CossSwapDir::optionBlockSizeDump(StoreEntry * e) const -{ - storeAppendPrintf(e, " block-size=%d", 1 << blksz_bits); -} - -StoreSearch * -CossSwapDir::search(String const url, HttpRequest *) -{ - if (url.size()) - fatal ("Cannot search by url yet\n"); - - return new StoreSearchCoss (this); -} - -char const * -CossSwapDir::stripePath() const -{ - if (!stripe_path) { - String result = path; - result.append("/stripe"); - const_cast(this)->stripe_path = xstrdup(result.termedBuf()); - } - - return stripe_path; -} - -CBDATA_CLASS_INIT(StoreSearchCoss); -StoreSearchCoss::StoreSearchCoss(RefCount aSwapDir) : sd(aSwapDir), callback (NULL), cbdata(NULL), _done (false), current(NULL), next_(sd->cossindex.tail) -{ - /* TODO: this races with the store as does the cleanlog stuff. - * FIXME by making coss_nodes ref counted */ -} - -/* do not link -StoreSearchCoss::StoreSearchCoss(StoreSearchCoss const &); -*/ - -StoreSearchCoss::~StoreSearchCoss() -{} - -void -StoreSearchCoss::next(void (callback)(void *cbdata), void *cbdata) -{ - next(); - callback (cbdata); -} - -bool -StoreSearchCoss::next() -{ - current = next_; - - if (next_) - next_ = next_->prev; - - if (!current) - _done = true; - - return current != NULL; -} - -bool -StoreSearchCoss::error() const -{ - return false; -} - -bool -StoreSearchCoss::isDone() const -{ - return _done; -} - -StoreEntry * -StoreSearchCoss::currentItem() -{ - if (!current) - return NULL; - - return static_cast( current->data ); -} diff --git a/src/fs/coss/store_io_coss.cc b/src/fs/coss/store_io_coss.cc deleted file mode 100644 index 95b8d6ef39..0000000000 --- a/src/fs/coss/store_io_coss.cc +++ /dev/null @@ -1,605 +0,0 @@ - -/* - * DEBUG: section 79 Storage Manager COSS Interface - * AUTHOR: Eric Stern - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid.h" -#include "CossSwapDir.h" -#include "DiskIO/DiskIOStrategy.h" -#include "fde.h" -#include "MemObject.h" -#include "SquidConfig.h" -#include "store_coss.h" -#include "Store.h" -#include "StoreFScoss.h" -#include "SwapDir.h" - -CBDATA_TYPE(CossMemBuf); - -/* === PUBLIC =========================================================== */ - -CossState::CossState(CossSwapDir *aCSD):SD (aCSD) -{} - -/* - * This routine sucks. I want to rewrite it when possible, and I also think - * that we should check after creatmembuf() to see if the object has a - * RELEASE_REQUEST set on it (thanks Eric!) rather than this way which seems - * to work.. - * -- Adrian - */ -sfileno -CossSwapDir::allocate(const StoreEntry * e, int which) -{ - CossMemBuf *newmb; - off_t retofs; - off_t allocsize; - int coll = 0; - sfileno checkf; - - /* Make sure we chcek collisions if reallocating */ - - if (which == COSS_ALLOC_REALLOC) { - checkf = e->swap_filen; - ++ StoreFScoss::GetInstance().stats.alloc.realloc; - } else { - checkf = -1; - ++ StoreFScoss::GetInstance().stats.alloc.alloc; - } - - if (e->swap_file_sz > 0) - allocsize = e->swap_file_sz; - else - allocsize = e->objectLen() + e->mem_obj->swap_hdr_sz; - - /* Check if we have overflowed the disk .. */ - if (current_offset + allocsize > static_cast(maxSize())) { - /* - * tried to allocate past the end of the disk, so wrap - * back to the beginning - */ - ++ StoreFScoss::GetInstance().stats.disk_overflows; - current_membuf->flags.full = true; - current_membuf->diskend = current_offset; - current_membuf->maybeWrite(this); - current_offset = 0; /* wrap back to beginning */ - debugs(79, 2, "CossSwapDir::allocate: wrap to 0"); - - newmb = createMemBuf(0, checkf, &coll); - current_membuf = newmb; - - /* Check if we have overflowed the MemBuf */ - } else if ((current_offset + allocsize) >= current_membuf->diskend) { - /* - * Skip the blank space at the end of the stripe. start over. - */ - ++ StoreFScoss::GetInstance().stats.stripe_overflows; - current_membuf->flags.full = true; - current_offset = current_membuf->diskend; - current_membuf->maybeWrite(this); - debugs(79, 2, "CossSwapDir::allocate: New offset - " << current_offset); - newmb = createMemBuf(current_offset, checkf, &coll); - current_membuf = newmb; - } - - /* - * If we didn't get a collision, then update the current offset - * and return it - */ - if (coll == 0) { - retofs = current_offset; - current_offset = retofs + allocsize; - /* round up to our blocksize */ - current_offset = ((current_offset + blksz_mask) >> blksz_bits ) << blksz_bits; - return storeCossDiskOffsetToFileno(retofs); - } else { - ++ StoreFScoss::GetInstance().stats.alloc.collisions; - debugs(79, 3, "CossSwapDir::allocate: Collision"); - return -1; - } -} - -bool -CossSwapDir::unlinkdUseful() const -{ - // no entry-specific files to unlink - return false; -} - -void -CossSwapDir::unlink(StoreEntry & e) -{ - debugs(79, 3, "storeCossUnlink: offset " << e.swap_filen); - if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) { - cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); - --n_disk_objects; - } - ++ StoreFScoss::GetInstance().stats.unlink.ops; - ++ StoreFScoss::GetInstance().stats.unlink.success; - storeCossRemove(this, &e); -} - -StoreIOState::Pointer -CossSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * callback, void *callback_data) -{ - CossState *cstate; - StoreIOState::Pointer sio = new CossState(this); - cstate = dynamic_cast(sio.getRaw()); - sio->offset_ = 0; - sio->mode = O_WRONLY | O_BINARY; - - /* - * If we get handed an object with a size of -1, - * the squid code is broken - */ - assert(e.mem_obj->object_sz != -1); - ++ StoreFScoss::GetInstance().stats.create.ops; - - /* - * this one is kinda strange - Eric called allocate(), then - * storeCossOpen(O_RDONLY) .. weird. Anyway, I'm allocating this now. - */ - cstate->st_size = e.objectLen() + e.mem_obj->swap_hdr_sz; - sio->swap_dirn = index; - sio->swap_filen = allocate(&e, COSS_ALLOC_ALLOCATE); - debugs(79, 3, "storeCossCreate: offset " << - storeCossFilenoToDiskOffset(sio->swap_filen) << - ", size " << (long int) cstate->st_size << ", end " << - (sio->swap_filen + cstate->st_size)); - - /* assume allocate() always succeeds */ - assert(-1 != sio->swap_filen); - - sio->callback = callback; - sio->file_callback = file_callback; - sio->callback_data = cbdataReference(callback_data); - sio->e = &e; - - cstate->flags.writing = false; - cstate->flags.reading = false; - cstate->readbuffer = NULL; - cstate->reqdiskoffset = -1; - - /* Now add it into the index list */ - storeCossAdd(this, &e); - - cstate->lockMemBuf(); - ++ StoreFScoss::GetInstance().stats.create.success; - return sio; -} - -StoreIOState::Pointer -CossSwapDir::openStoreIO(StoreEntry & e, StoreIOState::STFNCB * file_callback, - StoreIOState::STIOCB * callback, void *callback_data) -{ - char *p; - CossState *cstate; - sfileno f = e.swap_filen; - - debugs(79, 3, "storeCossOpen: offset " << f); - ++ StoreFScoss::GetInstance().stats.open.ops; - - StoreIOState::Pointer sio = new CossState (this); - cstate = dynamic_cast(sio.getRaw()); - - sio->swap_filen = f; - sio->swap_dirn = index; - sio->offset_ = 0; - sio->mode = O_RDONLY | O_BINARY; - sio->callback = callback; - sio->file_callback = file_callback; - sio->callback_data = cbdataReference(callback_data); - cstate->st_size = e.swap_file_sz; - sio->e = &e; - - cstate->flags.writing = false; - cstate->flags.reading = false; - cstate->readbuffer = NULL; - cstate->reqdiskoffset = -1; - p = storeCossMemPointerFromDiskOffset(storeCossFilenoToDiskOffset(f), NULL); - /* make local copy so we don't have to lock membuf */ - - if (p) { - cstate->readbuffer = (char *)xmalloc(cstate->st_size); - memcpy(cstate->readbuffer, p, cstate->st_size); - ++ StoreFScoss::GetInstance().stats.open_mem_hits; - } else { - /* Do the allocation */ - /* this is the first time we've been called on a new sio - * read the whole object into memory, then return the - * requested amount - */ - ++ StoreFScoss::GetInstance().stats.open_mem_misses; - /* - * This bit of code actually does the LRU disk thing - we realloc - * a place for the object here, and the file_read() reads the object - * into the cossmembuf for later writing .. - */ - cstate->reqdiskoffset = storeCossFilenoToDiskOffset(sio->swap_filen); - sio->swap_filen = -1; - sio->swap_filen = allocate(&e, COSS_ALLOC_REALLOC); - - if (sio->swap_filen == -1) { - /* We have to clean up neatly .. */ - ++ StoreFScoss::GetInstance().stats.open.fail; - ++numcollisions; - debugs(79, 2, "storeCossOpen: Reallocation of " << e.swap_dirn << "/" << e.swap_filen << " failed"); - /* XXX XXX XXX Will squid call storeUnlink for this object? */ - return NULL; - } - - /* Notify the upper levels that we've changed file number */ - sio->file_callback(sio->callback_data, 0, sio); - - /* - * lock the buffer so it doesn't get swapped out on us - * this will get unlocked in storeCossClose - */ - cstate->lockMemBuf(); - - /* - * Do the index magic to keep the disk and memory LRUs identical - */ - storeCossRemove(this, &e); - - storeCossAdd(this, &e); - - /* - * NOTE cstate->readbuffer is NULL. We'll actually read - * the disk data into the MemBuf in storeCossRead() and - * return that pointer back to the caller - */ - } - - ++ StoreFScoss::GetInstance().stats.open.success; - return sio; -} - -/// COSS does not distinguish different closure types -void -CossState::close(int) -{ - debugs(79, 3, "storeCossClose: offset " << swap_filen); - - ++ StoreFScoss::GetInstance().stats.close.ops; - ++ StoreFScoss::GetInstance().stats.close.success; - SD->storeCossMemBufUnlock(this); - doCallback(0); -} - -void -CossState::read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data) -{ - char *p; - CossSwapDir *SD = (CossSwapDir *)INDEXSD(swap_dirn); - - ++ StoreFScoss::GetInstance().stats.read.ops; - assert(read.callback == NULL); - assert(read.callback_data == NULL); - read.callback = callback; - read.callback_data = cbdataReference(callback_data); - debugs(79, 3, "storeCossRead: offset " << offset); - offset_ = offset; - flags.reading = true; - - if ((offset + (off_t)size) > st_size) - size = st_size - offset; - - requestlen = size; - - requestbuf = buf; - - requestoffset = offset; - - if (readbuffer == NULL) { - p = SD->storeCossMemPointerFromDiskOffset(SD->storeCossFilenoToDiskOffset(swap_filen), NULL); - sfileno tempReqdiskoffset = reqdiskoffset; - reqdiskoffset = 0; /* XXX */ - SD->theFile->read(new CossRead(ReadRequest(p, st_size, tempReqdiskoffset), this)); - } else { - /* - * It was copied from memory in storeCossOpen() - */ - ReadRequest::Pointer readRequest = new CossRead(ReadRequest( - (char *)readbuffer,st_size, 0), this); - SD->readCompleted(readbuffer, st_size, 0, readRequest); - } -} - -void -CossState::write(char const *buf, size_t size, off_t offset, FREE * free_func) -{ - char *dest; - CossMemBuf *membuf; - off_t diskoffset; - - /* - * If we get handed an object with a size of -1, - * the squid code is broken - */ - assert(e->mem_obj->object_sz != -1); - ++ StoreFScoss::GetInstance().stats.write.ops; - - debugs(79, 3, "storeCossWrite: offset " << offset_ << ", len " << (unsigned long int) size); - diskoffset = SD->storeCossFilenoToDiskOffset(swap_filen) + offset_; - CossSwapDir *SD = (CossSwapDir *)INDEXSD(swap_dirn); - dest = SD->storeCossMemPointerFromDiskOffset(diskoffset, &membuf); - assert(dest != NULL); - memcpy(dest, buf, size); - offset_ += size; - - if (free_func) - (free_func) ((char *)buf); - - ++ StoreFScoss::GetInstance().stats.write.success; -} - -off_t -CossSwapDir::storeCossFilenoToDiskOffset(sfileno f) -{ - return (off_t) f << blksz_bits; -} - -sfileno -CossSwapDir::storeCossDiskOffsetToFileno(off_t o) -{ - assert(0 == (o & blksz_mask)); - return o >> blksz_bits; -} - -CossMemBuf * -CossSwapDir::storeCossFilenoToMembuf(sfileno f) -{ - CossMemBuf *t = NULL; - dlink_node *m; - off_t o = storeCossFilenoToDiskOffset(f); - - for (m = membufs.head; m; m = m->next) { - t = (CossMemBuf *)m->data; - - if ((o >= (off_t)t->diskstart) && (o < (off_t)t->diskend)) - break; - } - - assert(t); - return t; -} - -/* === STATIC =========================================================== */ - -CBDATA_CLASS_INIT(CossRead); -CBDATA_CLASS_INIT(CossWrite); - -void -CossState::doCallback(int errflag) -{ - STIOCB *callback = this->callback; - void *cbdata; - debugs(79, 3, "CossState::doCallback: errflag=" << errflag); - assert(NULL == locked_membuf); - xfree(readbuffer); - this->callback = NULL; - - if (cbdataReferenceValidDone(callback_data, &cbdata)) - callback(cbdata, errflag, this); -} - -char * -CossSwapDir::storeCossMemPointerFromDiskOffset(off_t offset, CossMemBuf ** mb) -{ - CossMemBuf *t; - dlink_node *m; - - for (m = membufs.head; m; m = m->next) { - t = (CossMemBuf *)m->data; - - if ((offset >= t->diskstart) && (offset < t->diskend)) { - if (mb) - *mb = t; - - return &t->buffer[offset - t->diskstart]; - } - } - - if (mb) - *mb = NULL; - - return NULL; -} - -void -CossState::lockMemBuf() -{ - CossMemBuf *t = SD->storeCossFilenoToMembuf(swap_filen); - debugs(79, 3, "CossState::lockMemBuf: locking " << t << ", lockcount " << t->lockcount); - locked_membuf = t; - ++t->lockcount; -} - -void -CossSwapDir::storeCossMemBufUnlock(StoreIOState::Pointer sio) -{ - CossMemBuf *t = storeCossFilenoToMembuf(sio->swap_filen); - CossState *cstate = dynamic_cast(sio.getRaw()); - - if (NULL == t) - return; - - debugs(79, 3, "storeCossMemBufUnlock: unlocking " << t << ", lockcount " << t->lockcount); - - -- t->lockcount; - - cstate->locked_membuf = NULL; - - t->maybeWrite(this); -} - -void -CossSwapDir::sync() -{ - CossMemBuf *t; - dlink_node *m; - off_t end; - - /* First, flush pending IO ops */ - io->sync(); - - /* Then, flush any in-memory partial membufs */ - - if (!membufs.head) - return; - - for (m = membufs.head; m; m = m->next) { - t = (CossMemBuf *)m->data; - - if (t->flags.writing) { - debugs(79, DBG_IMPORTANT, "WARNING: sleeping for 5 seconds in storeCossSync()"); - sleep(5); /* XXX EEEWWW! */ - } - - end = (t == current_membuf) ? current_offset : t->diskend; - - if (end > t->diskstart) - theFile->write(new CossWrite(WriteRequest((char const *)&t->buffer, t->diskstart, end - t->diskstart, NULL), t)); - - /* and flush */ - io->sync(); - } -} - -void -CossMemBuf::maybeWrite(CossSwapDir * SD) -{ - describe(3, __LINE__); - - if (!flags.full) - debugs(79, 3, "membuf " << this << " not full"); - else if (flags.writing) - debugs(79, 3, "membuf " << this << " writing"); - else if (lockcount) - debugs(79, 3, "membuf " << this << " lockcount=" << lockcount); - else - write(SD); -} - -void -CossMemBuf::write(CossSwapDir * SD) -{ - ++ StoreFScoss::GetInstance().stats.stripe_write.ops; - debugs(79, 3, "CossMemBuf::write: offset " << diskstart << ", len " << (diskend - diskstart)); - flags.writing = true; - /* XXX Remember that diskstart/diskend are block offsets! */ - SD->theFile->write(new CossWrite(WriteRequest((char const *)&buffer, diskstart, diskend - diskstart, NULL), this)); -} - -CossMemBuf * -CossSwapDir::createMemBuf(off_t start, sfileno curfn, int *collision) -{ - CossMemBuf *newmb; - CossMemBuf *t; - StoreEntry *e; - dlink_node *m, *prev; - int numreleased = 0; - - CBDATA_INIT_TYPE_FREECB(CossMemBuf, NULL); - newmb = cbdataAlloc(CossMemBuf); - newmb->diskstart = start; - debugs(79, 3, "CossSwapDir::createMemBuf: creating new membuf at " << newmb->diskstart); - debugs(79, 3, "CossSwapDir::createMemBuf: at " << newmb); - newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ; - newmb->lockcount = 0; - newmb->SD = this; - /* XXX This should be reversed, with the new buffer last in the chain */ - dlinkAdd(newmb, &newmb->node, &membufs); - - /* Print out the list of membufs */ - - debugs(79, 3, "CossSwapDir::createMemBuf: membuflist:"); - - for (m = membufs.head; m; m = m->next) { - t = (CossMemBuf *)m->data; - t->describe(3, __LINE__); - } - - /* - * Kill objects from the tail to make space for a new chunk - */ - for (m = cossindex.tail; m; m = prev) { - off_t o; - prev = m->prev; - e = (StoreEntry *)m->data; - o = storeCossFilenoToDiskOffset(e->swap_filen); - - if (curfn == e->swap_filen) - *collision = 1; /* Mark an object alloc collision */ - - if ((o >= (off_t)newmb->diskstart) && (o < (off_t)newmb->diskend)) { - e->release(); - ++numreleased; - } else - break; - } - - if (numreleased > 0) - debugs(79, 3, "CossSwapDir::createMemBuf: this allocation released " << numreleased << " storeEntries"); - - ++ StoreFScoss::GetInstance().stats.stripes; - - return newmb; -} - -/* - * Creates the initial membuf after rebuild - */ -void -CossSwapDir::startMembuf() -{ - CossMemBuf *newmb; - newmb = createMemBuf(current_offset, -1, NULL); - assert(!current_membuf); - current_membuf = newmb; -} - -/* - * Clean up any references from the SIO before it get's released. - */ -CossState::~CossState() -{} - -void -CossMemBuf::describe(int level, int line) -{ - debugs(79, level, "membuf " << this << ", LC:" << std::setfill('0') << - std::setw(2) << lockcount << ", ST:" << - std::setw(10) << (unsigned long) diskstart << ", FL:" << - (flags.full ? 'F' : '.') << (flags.writing ? 'W' : '.')); -} - diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc index bb29426a0c..7f4526b367 100644 --- a/src/fs/rock/RockSwapDir.cc +++ b/src/fs/rock/RockSwapDir.cc @@ -146,7 +146,7 @@ Rock::SwapDir::entryLimitAllowed() const return min(max(eLimitLo, eWanted), entryLimitHigh()); } -// TODO: encapsulate as a tool; identical to CossSwapDir::create() +// TODO: encapsulate as a tool void Rock::SwapDir::create() { diff --git a/src/tests/testCoss.cc b/src/tests/testCoss.cc deleted file mode 100644 index ffec2b4c90..0000000000 --- a/src/tests/testCoss.cc +++ /dev/null @@ -1,298 +0,0 @@ -#define SQUID_UNIT_TEST 1 -#include "squid.h" -#include "ConfigParser.h" -#include "testCoss.h" -#include "Store.h" -#include "SwapDir.h" -#include "DiskIO/DiskIOModule.h" -#include "fs/coss/CossSwapDir.h" -#include "Mem.h" -#include "MemObject.h" -#include "HttpHeader.h" -#include "HttpReply.h" -#include "RequestFlags.h" -#include "StoreFileSystem.h" -#include "testStoreSupport.h" -#include "SquidConfig.h" - -#if HAVE_STDEXCEPT -#include -#endif - -#define TESTDIR "testCoss_Store" - -CPPUNIT_TEST_SUITE_REGISTRATION( testCoss ); - -typedef RefCount SwapDirPointer; -extern REMOVALPOLICYCREATE createRemovalPolicy_lru; - -static void -addSwapDir(SwapDirPointer aStore) -{ - allocate_new_swapdir(&Config.cacheSwap); - Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); - ++Config.cacheSwap.n_configured; -} - -void -testCoss::commonInit() -{ - static bool inited = false; - - if (inited) - return; - - StoreFileSystem::SetupAllFs(); - - Config.Store.avgObjectSize = 1024; - - Config.Store.objectsPerBucket = 20; - - Config.Store.maxObjectSize = 2048; - - Config.store_dir_select_algorithm = xstrdup("round-robin"); - - Config.replPolicy = new RemovalPolicySettings; - - Config.replPolicy->type = xstrdup ("lru"); - - Config.replPolicy->args = NULL; - - /* garh garh */ - storeReplAdd("lru", createRemovalPolicy_lru); - - visible_appname_string = xstrdup(APP_FULLNAME); - - Mem::Init(); - - comm_init(); - - httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ - - httpReplyInitModule(); /* must go before accepting replies */ - - mem_policy = createRemovalPolicy(Config.replPolicy); - - inited = true; -} - -void -testCoss::testCossCreate() -{ - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); - - Store::Root(new StoreController); - - SwapDirPointer aStore (new CossSwapDir()); - - addSwapDir(aStore); - - commonInit(); - - char *path=xstrdup(TESTDIR); - - char *config_line=xstrdup("100 max-size=102400 block-size=512 IOEngine=Blocking"); - - ConfigParser::SetCfgLine(config_line); - - aStore->parse(0, path); - - safe_free(path); - - safe_free(config_line); - - /* ok, ready to create */ - aStore->create(); - - struct stat sb; - - CPPUNIT_ASSERT(::stat(TESTDIR, &sb) == 0); - - /* TODO: check the size */ - - Store::Root(NULL); - - free_cachedir(&Config.cacheSwap); - - /* todo: here we should test a dirty rebuild */ - - // safe_free(Config.replPolicy->type); - // delete Config.replPolicy; - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); -} - -/* TODO make this a cbdata class */ - -static bool cbcalled; - -static void -searchCallback(void *cbdata) -{ - cbcalled = true; -} - -void -testCoss::testCossSearch() -{ - /* test sequence - * make a valid working ufs swapdir - * put two entries in it and sync logs - * search the ufs dir - * check the entries we find are what we want - */ - - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); - - Store::Root(new StoreController); - - SwapDirPointer aStore (new CossSwapDir()); - - addSwapDir(aStore); - - commonInit(); - - char *path=xstrdup(TESTDIR); - - char *config_line=xstrdup("100 max-size=102400 block-size=512 IOEngine=Blocking"); - - ConfigParser::SetCfgLine(config_line); - - aStore->parse(0, path); - - safe_free(path); - - safe_free(config_line); - - /* ok, ready to create */ - aStore->create(); - - /* ok, ready to use */ - Store::Root().init(); - - /* rebuild is a scheduled event */ - StockEventLoop loop; - - /* our swapdir must be scheduled to rebuild */ - CPPUNIT_ASSERT_EQUAL(2, StoreController::store_dirs_rebuilding); - - loop.run(); - - /* cannot use loop.run(); as the loop will never idle: the store-dir - * clean() scheduled event prevents it - */ - - /* nothing left to rebuild */ - CPPUNIT_ASSERT_EQUAL(1, StoreController::store_dirs_rebuilding); - - /* add an entry */ - { - /* Create "vary" base object */ - RequestFlags flags; - flags.cachable = true; - StoreEntry *pe = storeCreateEntry("dummy url", "dummy log url", flags, METHOD_GET); - HttpReply *rep = (HttpReply *) pe->getReply(); // bypass const - rep->setHeaders(Http::scOkay, "dummy test object", "x-squid-internal/test", -1, -1, squid_curtime + 100000); - - pe->setPublicKey(); - - pe->buffer(); - /* TODO: remove this when the metadata is separated */ - { - Packer p; - packerToStoreInit(&p, pe); - pe->getReply()->packHeadersInto(&p); - packerClean(&p); - } - - pe->flush(); - pe->timestampsSet(); - pe->complete(); - pe->swapOut(); - CPPUNIT_ASSERT(pe->swap_dirn == 0); - CPPUNIT_ASSERT(pe->swap_filen == 0); - pe->unlock(); - } - - storeDirWriteCleanLogs(0); - - /* here we cheat: we know that UFSSwapDirs search off disk. If we did an init call to a new - * swapdir instance, we'd not be testing a clean build. - */ - StoreSearchPointer search = aStore->search (NULL, NULL); /* search for everything in the store */ - - /* nothing should be immediately available */ -#if 0 - - CPPUNIT_ASSERT(search->next() == false); -#endif - - CPPUNIT_ASSERT(search->error() == false); - CPPUNIT_ASSERT(search->isDone() == false); - CPPUNIT_ASSERT(search->currentItem() == NULL); - - /* trigger a callback */ - cbcalled = false; - search->next(searchCallback, NULL); - CPPUNIT_ASSERT(cbcalled == true); - - /* we should have access to a entry now, that matches the entry we had before */ - //CPPUNIT_ASSERT(search->next() == false); - CPPUNIT_ASSERT(search->error() == false); - CPPUNIT_ASSERT(search->isDone() == false); - CPPUNIT_ASSERT(search->currentItem() != NULL); - - /* trigger another callback */ - cbcalled = false; - search->next(searchCallback, NULL); - CPPUNIT_ASSERT(cbcalled == true); - - /* now we should have no error, we should have finished and have no current item */ - //CPPUNIT_ASSERT(search->next() == false); - CPPUNIT_ASSERT(search->error() == false); - CPPUNIT_ASSERT(search->isDone() == true); - CPPUNIT_ASSERT(search->currentItem() == NULL); - - Store::Root(NULL); - - free_cachedir(&Config.cacheSwap); - - /* todo: here we should test a dirty rebuild */ - - //TODO: do this once, or each time. safe_free(Config.replPolicy->type); - // delete Config.replPolicy; - - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); -} - -/* The COSS store should always configure an IO engine even if none is - * supplied on the configuration line. - */ -void -testCoss::testDefaultEngine() -{ - /* boring common test boilerplate */ - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); - - Store::Root(new StoreController); - SwapDirPointer aStore (new CossSwapDir()); - addSwapDir(aStore); - commonInit(); - - char *path=xstrdup(TESTDIR); - char *config_line=xstrdup("100 max-size=102400 block-size=512"); - ConfigParser::SetCfgLine(config_line); - aStore->parse(0, path); - safe_free(path); - safe_free(config_line); - CPPUNIT_ASSERT(aStore->io != NULL); - - Store::Root(NULL); - free_cachedir(&Config.cacheSwap); - if (0 > system ("rm -rf " TESTDIR)) - throw std::runtime_error("Failed to clean test work directory"); -} diff --git a/src/tests/testCoss.h b/src/tests/testCoss.h deleted file mode 100644 index 70a159a35b..0000000000 --- a/src/tests/testCoss.h +++ /dev/null @@ -1,29 +0,0 @@ - -#ifndef SQUID_SRC_TEST_STORECONTROLLER_H -#define SQUID_SRC_TEST_STORECONTROLLER_H - -#include - -/* - * test the store framework - */ - -class testCoss : public CPPUNIT_NS::TestFixture -{ - CPPUNIT_TEST_SUITE( testCoss ); - CPPUNIT_TEST( testCossCreate ); - CPPUNIT_TEST( testCossSearch ); - CPPUNIT_TEST( testDefaultEngine ); - CPPUNIT_TEST_SUITE_END(); - -public: - -protected: - void commonInit(); - void testCossCreate(); - void testCossSearch(); - void testDefaultEngine(); -}; - -#endif - diff --git a/src/tests/testDiskIO.cc b/src/tests/testDiskIO.cc index 38dbc6252a..7ba797b177 100644 --- a/src/tests/testDiskIO.cc +++ b/src/tests/testDiskIO.cc @@ -12,11 +12,6 @@ #include "StoreFileSystem.h" #include "testStoreSupport.h" -#if 0 -// AYJ: COSS in Squid-3 is disabled. -#include "fs/coss/CossSwapDir.h" -#endif - #if HAVE_STDEXCEPT #include #endif diff --git a/test-suite/Makefile.am b/test-suite/Makefile.am index 2760a8cbf6..e8c88c8f0a 100644 --- a/test-suite/Makefile.am +++ b/test-suite/Makefile.am @@ -129,3 +129,5 @@ squid-conf-tests: $(top_builddir)/src/squid.conf.default $(srcdir)/squidconf/* else break; fi; \ done; \ if test "$$failed" -eq 0; then cp $(TRUE) $@ ; fi + +CLEANFILES += squid-conf-tests diff --git a/test-suite/buildtests/layer-01-minimal.opts b/test-suite/buildtests/layer-01-minimal.opts index 3e9096c75a..33b21c52dd 100644 --- a/test-suite/buildtests/layer-01-minimal.opts +++ b/test-suite/buildtests/layer-01-minimal.opts @@ -20,7 +20,6 @@ MAKETEST="distcheck" # # --without-default-user \ # --without-aufs-threads \ -# --without-coss-membuf-size \ # --without-filedescriptors \ # --without-cppunit-basedir \ # --without-build-environment \ diff --git a/test-suite/buildtests/layer-02-maximus.opts b/test-suite/buildtests/layer-02-maximus.opts index 8e564d05f6..0a617e5441 100644 --- a/test-suite/buildtests/layer-02-maximus.opts +++ b/test-suite/buildtests/layer-02-maximus.opts @@ -22,7 +22,6 @@ MAKETEST="distcheck" # # --with-default-user=NAME \ # --with-aufs-threads=N \ -# --with-coss-membuf-size=N \ # --with-filedescriptors=N \ # --with-cppunit-basedir=PATH \ # --with-openssl=PATH \ diff --git a/test-suite/buildtests/layer-04-noauth-everything.opts b/test-suite/buildtests/layer-04-noauth-everything.opts index 44f70c0b5a..96ebdca3b5 100644 --- a/test-suite/buildtests/layer-04-noauth-everything.opts +++ b/test-suite/buildtests/layer-04-noauth-everything.opts @@ -22,7 +22,6 @@ MAKETEST="distcheck" # # --with-default-user=NAME \ # --with-aufs-threads=N \ -# --with-coss-membuf-size=N \ # --with-filedescriptors=N \ # --with-cppunit-basedir=PATH \ # --with-openssl=PATH \ -- 2.47.2