From: Alex Rousskov Date: Fri, 14 Oct 2011 15:53:02 +0000 (-0600) Subject: Add basic "make check" tests for Rock Store, based on UFS and COSS tests. X-Git-Tag: BumpSslServerFirst.take01~98 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9bb016119a1ac45a31f638fd275a0adf8b2fc844;p=thirdparty%2Fsquid.git Add basic "make check" tests for Rock Store, based on UFS and COSS tests. --- diff --git a/configure.ac b/configure.ac index 273d89ce84..586864c052 100644 --- a/configure.ac +++ b/configure.ac @@ -786,6 +786,7 @@ for fs in $squid_storeio_module_candidates none; do "x$squid_disk_module_candidates_Blocking" != "xyes"; then AC_MSG_ERROR([Storage module Rock requires IpcIo or Blocking DiskIO module]) fi + STORE_TESTS="$STORE_TESTS tests/testRock$EXEEXT" ;; ufs) STORE_TESTS="$STORE_TESTS tests/testUfs$EXEEXT" diff --git a/src/Makefile.am b/src/Makefile.am index 2f35ef1d54..9505971471 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -186,6 +186,7 @@ EXTRA_PROGRAMS = \ recv-announce \ tests/testUfs \ tests/testCoss \ + tests/testRock \ tests/testNull \ ufsdump @@ -2651,6 +2652,123 @@ tests_testUfs_LDFLAGS = $(LIBADD_DL) tests_testUfs_DEPENDENCIES = \ $(SWAP_TEST_DS) +tests_testRock_SOURCES = \ + cbdata.cc \ + CacheDigest.cc \ + ConfigOption.cc \ + ConfigParser.cc \ + disk.cc \ + ETag.cc \ + EventLoop.cc \ + event.cc \ + fd.cc \ + HttpBody.cc \ + HttpHdrCc.cc \ + HttpHdrContRange.cc \ + HttpHdrRange.cc \ + HttpHdrSc.cc \ + HttpHdrScTarget.cc \ + HttpHeader.cc \ + HttpHeaderTools.cc \ + HttpMsg.cc \ + HttpReply.cc \ + HttpRequestMethod.cc \ + HttpStatusLine.cc \ + int.cc \ + mem.cc \ + MemBuf.cc \ + MemObject.cc \ + mem_node.cc \ + Packer.cc \ + Parsing.cc \ + RemovalPolicy.cc \ + StatHist.cc \ + stmem.cc \ + store.cc \ + StoreFileSystem.cc \ + StoreIOState.cc \ + StoreMeta.cc \ + StoreMetaMD5.cc \ + StoreMetaSTD.cc \ + StoreMetaSTDLFS.cc \ + StoreMetaURL.cc \ + StoreMetaUnpacker.cc \ + StoreMetaVary.cc \ + store_dir.cc \ + store_io.cc \ + store_key_md5.cc \ + store_swapmeta.cc \ + store_swapout.cc \ + String.cc \ + SwapDir.cc \ + tests/testRock.cc \ + tests/testMain.cc \ + tests/testRock.h \ + tests/testStoreSupport.cc \ + tests/testStoreSupport.h \ + tests/stub_access_log.cc \ + tests/stub_cache_cf.cc \ + tests/stub_cache_manager.cc \ + tests/stub_client_db.cc \ + tests/stub_client_side_request.cc \ + tests/stub_debug.cc \ + tests/stub_errorpage.cc \ + tests/stub_HelperChildConfig.cc \ + tests/stub_http.cc \ + tests/stub_HttpRequest.cc \ + tests/stub_ipc.cc \ + tests/stub_MemStore.cc \ + tests/stub_mime.cc \ + tests/stub_Port.cc \ + tests/stub_pconn.cc \ + tests/stub_store_client.cc \ + tests/stub_store_rebuild.cc \ + tests/stub_tools.cc \ + tests/stub_UdsOp.cc \ + time.cc \ + url.cc \ + URLScheme.cc \ + wordlist.cc \ + $(DELAY_POOL_SOURCE) \ + $(DISKIO_SOURCE) \ + $(UNLINKDSOURCE) +nodist_tests_testRock_SOURCES = \ + $(DISKIO_GEN_SOURCE) \ + swap_log_op.cc \ + SquidMath.cc \ + SquidMath.h \ + $(TESTSOURCES) +tests_testRock_LDADD = \ + anyp/libanyp.la \ + libsquid.la \ + comm/libcomm.la \ + ip/libip.la \ + fs/libfs.la \ + $(AUTH_LIBS) \ + $(COMMON_LIBS) \ + $(REPL_OBJS) \ + $(DISK_LIBS) \ + $(DISK_OS_LIBS) \ + acl/libacls.la \ + acl/libapi.la \ + acl/libstate.la \ + eui/libeui.la \ + ipc/libipc.la \ + mgr/libmgr.la \ + base/libbase.la \ + $(SSL_LIBS) \ + $(top_builddir)/lib/libmisccontainers.la \ + $(top_builddir)/lib/libmiscencoding.la \ + $(top_builddir)/lib/libmiscutil.la \ + $(REGEXLIB) \ + $(SQUID_CPPUNIT_LIBS) \ + $(SSLLIB) \ + $(COMPAT_LIB) \ + $(XTRA_LIBS) +tests_testRock_LDFLAGS = $(LIBADD_DL) +tests_testRock_DEPENDENCIES = \ + $(SWAP_TEST_DS) + tests_testCoss_SOURCES = \ tests/testCoss.cc \ tests/testMain.cc \ diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc index 316c0a6fec..e062b1dc3e 100644 --- a/src/fs/rock/RockSwapDir.cc +++ b/src/fs/rock/RockSwapDir.cc @@ -5,7 +5,6 @@ */ #include "config.h" -#include "base/RunnersRegistry.h" #include "ConfigOption.h" #include "DiskIO/DiskIOModule.h" #include "DiskIO/DiskIOStrategy.h" @@ -811,24 +810,12 @@ Rock::SwapDir::statfs(StoreEntry &e) const } -/// initializes shared memory segments used by Rock::SwapDir -class RockSwapDirRr: public Ipc::Mem::RegisteredRunner +namespace Rock { -public: - /* RegisteredRunner API */ - virtual ~RockSwapDirRr(); - -protected: - virtual void create(const RunnerRegistry &); - -private: - Vector owners; -}; - -RunnerRegistrationEntry(rrAfterConfig, RockSwapDirRr); - + RunnerRegistrationEntry(rrAfterConfig, SwapDirRr); +} -void RockSwapDirRr::create(const RunnerRegistry &) +void Rock::SwapDirRr::create(const RunnerRegistry &) { Must(owners.empty()); for (int i = 0; i < Config.cacheSwap.n_configured; ++i) { @@ -840,7 +827,7 @@ void RockSwapDirRr::create(const RunnerRegistry &) } } -RockSwapDirRr::~RockSwapDirRr() +Rock::SwapDirRr::~SwapDirRr() { for (size_t i = 0; i < owners.size(); ++i) delete owners[i]; diff --git a/src/fs/rock/RockSwapDir.h b/src/fs/rock/RockSwapDir.h index e2b8e37c3a..0fbcada4ff 100644 --- a/src/fs/rock/RockSwapDir.h +++ b/src/fs/rock/RockSwapDir.h @@ -33,6 +33,8 @@ public: virtual uint64_t currentCount() const; virtual bool doReportStat() const; virtual void swappedOut(const StoreEntry &e); + virtual void create(); + virtual void parse(int index, char *path); int64_t entryLimitHigh() const { return SwapFilenMax; } ///< Core limit int64_t entryLimitAllowed() const; @@ -42,7 +44,6 @@ public: protected: /* protected ::SwapDir API */ virtual bool needsDiskStrand() const; - virtual void create(); virtual void init(); virtual ConfigOption *getOptionTree() const; virtual bool allowOptionReconfigure(const char *const option) const; @@ -62,7 +63,6 @@ protected: virtual void readCompleted(const char *buf, int len, int errflag, RefCount< ::ReadRequest>); virtual void writeCompleted(int errflag, size_t len, RefCount< ::WriteRequest>); - virtual void parse(int index, char *path); void parseSize(const bool reconfiguring); ///< parses anonymous cache_dir size option void validateOptions(); ///< warns of configuration problems; may quit bool parseTimeOption(char const *option, const char *value, int reconfiguring); @@ -96,6 +96,22 @@ private: static const int64_t HeaderSize; ///< on-disk db header size }; +/// initializes shared memory segments used by Rock::SwapDir +class SwapDirRr: public Ipc::Mem::RegisteredRunner +{ +public: + /* ::RegisteredRunner API */ + virtual ~SwapDirRr(); + +protected: + /* Ipc::Mem::RegisteredRunner API */ + virtual void create(const RunnerRegistry &); + +private: + Vector owners; +}; + + } // namespace Rock #endif /* SQUID_FS_ROCK_SWAP_DIR_H */ diff --git a/src/tests/stub_store_rebuild.cc b/src/tests/stub_store_rebuild.cc index 22c567a0d9..608cbddeb2 100644 --- a/src/tests/stub_store_rebuild.cc +++ b/src/tests/stub_store_rebuild.cc @@ -36,9 +36,7 @@ void storeRebuildProgress(int sd_index, int total, int sofar) -{ - fatal ("Not implemented"); -} +{} void diff --git a/src/tests/stub_tools.cc b/src/tests/stub_tools.cc index f86d0d7605..27afd55be7 100644 --- a/src/tests/stub_tools.cc +++ b/src/tests/stub_tools.cc @@ -72,7 +72,10 @@ bool IamMasterProcess() { fprintf(stderr, "Not implemented"); - return false; + // Since most tests run as a single process, this is the best default. + // TODO: If some test case uses multiple processes and cares about + // its role, we may need to parameterize or remove this stub. + return true; } bool diff --git a/src/tests/testRock.cc b/src/tests/testRock.cc new file mode 100644 index 0000000000..4fbd233387 --- /dev/null +++ b/src/tests/testRock.cc @@ -0,0 +1,288 @@ +#define SQUID_UNIT_TEST 1 +#include "config.h" + +#include "DiskIO/DiskIOModule.h" +#include "HttpHeader.h" +#include "HttpReply.h" +#include "Mem.h" +#include "MemObject.h" +#include "Store.h" +#include "StoreFileSystem.h" +#include "StoreSearch.h" +#include "SwapDir.h" +#include "fs/rock/RockSwapDir.h" +#include "testRock.h" +#include "testStoreSupport.h" + +#if HAVE_STDEXCEPT +#include +#endif + +#define TESTDIR "testRock__testRockSearch" + +CPPUNIT_TEST_SUITE_REGISTRATION( testRock ); + +extern REMOVALPOLICYCREATE createRemovalPolicy_lru; + +static void +addSwapDir(testRock::SwapDirPointer aStore) +{ + allocate_new_swapdir(&Config.cacheSwap); + Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw(); + ++Config.cacheSwap.n_configured; +} + +void +testRock::setUp() +{ + CPPUNIT_NS::TestFixture::setUp(); + + if (0 > system ("rm -rf " TESTDIR)) + throw std::runtime_error("Failed to clean test work directory"); + + Store::Root(new StoreController); + + store = new Rock::SwapDir(); + + addSwapDir(store); + + commonInit(); + + char *path=xstrdup(TESTDIR); + + char *config_line=xstrdup("foo 100 max-size=16384"); + + strtok(config_line, w_space); + + store->parse(0, path); + + safe_free(path); + + safe_free(config_line); + + /* ok, ready to create */ + store->create(); + + rr = new Rock::SwapDirRr; + rr->run(rrAfterConfig); +} + +void +testRock::tearDown() +{ + CPPUNIT_NS::TestFixture::tearDown(); + + Store::Root(NULL); + + store = NULL; + + free_cachedir(&Config.cacheSwap); + + delete rr; + + // TODO: do this once, or each time. + // safe_free(Config.replPolicy->type); + // delete Config.replPolicy; + + if (0 > system ("rm -rf " TESTDIR)) + throw std::runtime_error("Failed to clean test work directory"); +} + +void +testRock::commonInit() +{ + static bool inited = false; + + if (inited) + return; + + StoreFileSystem::SetupAllFs(); + + Config.Store.avgObjectSize = 1024; + + Config.Store.objectsPerBucket = 20; + + Config.Store.maxObjectSize = 2048; + + Config.store_dir_select_algorithm = xstrdup("round-robin"); + + Config.replPolicy = new RemovalPolicySettings; + + Config.replPolicy->type = xstrdup ("lru"); + + Config.replPolicy->args = NULL; + + /* garh garh */ + storeReplAdd("lru", createRemovalPolicy_lru); + + visible_appname_string = xstrdup(APP_FULLNAME); + + Mem::Init(); + + comm_init(); + + httpHeaderInitModule(); /* must go before any header processing (e.g. the one in errorInitialize) */ + + httpReplyInitModule(); /* must go before accepting replies */ + + mem_policy = createRemovalPolicy(Config.replPolicy); + + inited = true; +} + +void +testRock::storeInit() +{ + /* ok, ready to use */ + Store::Root().init(); + + /* rebuild is a scheduled event */ + StockEventLoop loop; + + /* our swapdir must be scheduled to rebuild */ + CPPUNIT_ASSERT_EQUAL(2, StoreController::store_dirs_rebuilding); + + loop.run(); + + /* cannot use loop.run(); as the loop will never idle: the store-dir + * clean() scheduled event prevents it + */ + + /* nothing left to rebuild */ + CPPUNIT_ASSERT_EQUAL(1, StoreController::store_dirs_rebuilding); +} + +StoreEntry * +testRock::createEntry(const int i) +{ + request_flags flags; + flags.cachable = 1; + char url[64]; + snprintf(url, sizeof(url), "dummy url %i", i); + url[sizeof(url) - 1] = '\0'; + StoreEntry *const pe = + storeCreateEntry(url, "dummy log url", flags, METHOD_GET); + HttpReply *const rep = const_cast(pe->getReply()); + rep->setHeaders(HTTP_OK, "dummy test object", "x-squid-internal/test", + -1, -1, squid_curtime + 100000); + + pe->setPublicKey(); + + return pe; +} + +StoreEntry * +testRock::addEntry(const int i) +{ + StoreEntry *const pe = createEntry(i); + + pe->buffer(); + /* TODO: remove this when the metadata is separated */ + { + Packer p; + packerToStoreInit(&p, pe); + pe->getReply()->packHeadersInto(&p); + packerClean(&p); + } + + pe->flush(); + pe->timestampsSet(); + pe->complete(); + pe->swapOut(); + + return pe; +} + +StoreEntry * +testRock::getEntry(const int i) +{ + StoreEntry *const pe = createEntry(i); + return store->get(reinterpret_cast(pe->key)); +} + +void +testRock::testRockCreate() +{ + struct stat sb; + + CPPUNIT_ASSERT(::stat(TESTDIR, &sb) == 0); + + /* TODO: check the size */ + + /* TODO: test rebuild */ +} + +void +testRock::testRockSwapOut() +{ + storeInit(); + + // add few entries to prime the database + for (int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT_EQUAL((uint64_t)i, store->currentCount()); + + StoreEntry *const pe = addEntry(i); + + CPPUNIT_ASSERT(pe->swap_status == SWAPOUT_WRITING); + CPPUNIT_ASSERT(pe->swap_dirn == 0); + CPPUNIT_ASSERT(pe->swap_filen >= 0); + + // Rock::IoState::finishedWriting() schedules an AsyncCall + // storeSwapOutFileClosed(). Let it fire. + StockEventLoop loop; + loop.run(); + + CPPUNIT_ASSERT(pe->swap_status == SWAPOUT_DONE); + + pe->unlock(); + } + + CPPUNIT_ASSERT_EQUAL((uint64_t)5, store->currentCount()); + + // try to swap out entry to a used unlocked slot + { + StoreEntry *const pe = addEntry(4); + + CPPUNIT_ASSERT(pe->swap_status == SWAPOUT_WRITING); + CPPUNIT_ASSERT(pe->swap_dirn == 0); + CPPUNIT_ASSERT(pe->swap_filen >= 0); + + StockEventLoop loop; + loop.run(); + + CPPUNIT_ASSERT(pe->swap_status == SWAPOUT_DONE); + } + + // try to swap out entry to a used locked slot + { + StoreEntry *const pe = addEntry(5); + + CPPUNIT_ASSERT(pe->swap_status == SWAPOUT_WRITING); + CPPUNIT_ASSERT(pe->swap_dirn == 0); + CPPUNIT_ASSERT(pe->swap_filen >= 0); + + // the slot is locked here because the async calls have not run yet + StoreEntry *const pe2 = addEntry(5); + CPPUNIT_ASSERT(pe2->swap_status == SWAPOUT_NONE); + CPPUNIT_ASSERT(pe2->mem_obj->swapout.decision == + MemObject::SwapOut::swImpossible); + CPPUNIT_ASSERT(pe2->swap_dirn == -1); + CPPUNIT_ASSERT(pe2->swap_filen == -1); + + StockEventLoop loop; + loop.run(); + } + + CPPUNIT_ASSERT_EQUAL((uint64_t)6, store->currentCount()); + + // try to get and unlink entries + for (int i = 0; i < 6; ++i) { + StoreEntry *const pe = getEntry(i); + CPPUNIT_ASSERT(pe != NULL); + + pe->unlink(); + + StoreEntry *const pe2 = getEntry(i); + CPPUNIT_ASSERT(pe2 == NULL); + } +} diff --git a/src/tests/testRock.h b/src/tests/testRock.h new file mode 100644 index 0000000000..2951d8ff42 --- /dev/null +++ b/src/tests/testRock.h @@ -0,0 +1,37 @@ +#ifndef SQUID_SRC_TEST_TESTROCK_H +#define SQUID_SRC_TEST_TESTROCK_H + +#include + +/* + * test the store framework + */ + +class testRock : public CPPUNIT_NS::TestFixture +{ + CPPUNIT_TEST_SUITE( testRock ); + CPPUNIT_TEST( testRockCreate ); + CPPUNIT_TEST( testRockSwapOut ); + CPPUNIT_TEST_SUITE_END(); + +public: + virtual void setUp(); + virtual void tearDown(); + + typedef RefCount SwapDirPointer; + +protected: + void commonInit(); + void storeInit(); + StoreEntry *createEntry(const int i); + StoreEntry *addEntry(const int i); + StoreEntry *getEntry(const int i); + void testRockCreate(); + void testRockSwapOut(); + +private: + SwapDirPointer store; + Rock::SwapDirRr *rr; +}; + +#endif /* SQUID_SRC_TEST_TESTROCK_H */