From: JINMEI Tatuya Date: Fri, 8 Mar 2013 23:15:44 +0000 (-0800) Subject: [2831] introduced file-mapped memory segment with some basic functionalities. X-Git-Tag: bind10-1.2.0beta1-release~457^2~123 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d46357aadcf46d1be0be68fd510d9065ecba2a57;p=thirdparty%2Fkea.git [2831] introduced file-mapped memory segment with some basic functionalities. committing mostly as a backup. --- diff --git a/src/lib/util/Makefile.am b/src/lib/util/Makefile.am index 3960a8b6e5..1fd2de46e6 100644 --- a/src/lib/util/Makefile.am +++ b/src/lib/util/Makefile.am @@ -18,6 +18,7 @@ libb10_util_la_SOURCES += interprocess_sync_file.h interprocess_sync_file.cc libb10_util_la_SOURCES += interprocess_sync_null.h interprocess_sync_null.cc libb10_util_la_SOURCES += memory_segment.h libb10_util_la_SOURCES += memory_segment_local.h memory_segment_local.cc +libb10_util_la_SOURCES += memory_segment_mapped.h memory_segment_mapped.cc libb10_util_la_SOURCES += range_utilities.h libb10_util_la_SOURCES += hash/sha1.h hash/sha1.cc libb10_util_la_SOURCES += encode/base16_from_binary.h diff --git a/src/lib/util/memory_segment.h b/src/lib/util/memory_segment.h index 664bd3c644..7d376a7f2a 100644 --- a/src/lib/util/memory_segment.h +++ b/src/lib/util/memory_segment.h @@ -15,11 +15,25 @@ #ifndef MEMORY_SEGMENT_H #define MEMORY_SEGMENT_H +#include + #include namespace isc { namespace util { +class MemorySegmentOpenError : public Exception { +public: + MemorySegmentOpenError(const char* file, size_t line, const char* what) : + isc::Exception(file, line, what) {} +}; + +class MemorySegmentGrown : public Exception { +public: + MemorySegmentGrown(const char* file, size_t line, const char* what) : + isc::Exception(file, line, what) {} +}; + /// \brief Memory Segment Class /// /// This class specifies an interface for allocating memory diff --git a/src/lib/util/memory_segment_mapped.cc b/src/lib/util/memory_segment_mapped.cc new file mode 100644 index 0000000000..8fcf6ab1e6 --- /dev/null +++ b/src/lib/util/memory_segment_mapped.cc @@ -0,0 +1,128 @@ +// Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC") +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include + +#include +#include +#include + +#include +#include +#include + +using boost::interprocess::managed_mapped_file; +using boost::interprocess::open_or_create; +using boost::interprocess::open_only; + +namespace isc { +namespace util { + +struct MemorySegmentMapped::Impl { + Impl(const std::string& filename, size_t initial_size) : + filename_(filename), + base_sgmt_(new managed_mapped_file(open_or_create, filename.c_str(), + initial_size)) + {} + + Impl(const std::string& filename) : + filename_(filename), + base_sgmt_(new managed_mapped_file(open_only, filename.c_str())) + {} + + // mapped file; remember it in case we need to grow it. + const std::string filename_; + + // actual Boost implementation of mapped segment. + boost::scoped_ptr base_sgmt_; +}; + +MemorySegmentMapped::MemorySegmentMapped(const std::string& filename) : + impl_(0) +{ + try { + impl_ = new Impl(filename); + } catch (const boost::interprocess::interprocess_exception& ex) { + isc_throw(MemorySegmentOpenError, + "failed to open mapped memory segment for " << filename + << ": " << ex.what()); + } +} + +MemorySegmentMapped::MemorySegmentMapped(const std::string& filename, + bool create, size_t initial_size) : + impl_(0) +{ + try { + if (create) { + impl_ = new Impl(filename, initial_size); + } else { + impl_ = new Impl(filename); + } + } catch (const boost::interprocess::interprocess_exception& ex) { + isc_throw(MemorySegmentOpenError, + "failed to open mapped memory segment for " << filename + << ": " << ex.what()); + } +} + +MemorySegmentMapped::~MemorySegmentMapped() { + delete impl_; +} + +void* +MemorySegmentMapped::allocate(size_t size) { + void* ptr = impl_->base_sgmt_->allocate(size, std::nothrow); + if (ptr) { + return (ptr); + } + + // Grow the mapped segment doubling the size until we have sufficient + // free memory in the revised segment for the requested size. + while (impl_->base_sgmt_->get_free_memory() < size) { + // We first need to unmap it before calling grow(). + const size_t prev_size = impl_->base_sgmt_->get_size(); + impl_->base_sgmt_.reset(); + + const size_t new_size = prev_size * 2; + assert(new_size != 0); // assume grow fails before size overflow + + // TBD error handling + managed_mapped_file::grow(impl_->filename_.c_str(), + new_size - prev_size); + impl_->base_sgmt_.reset( + new managed_mapped_file(open_only, impl_->filename_.c_str())); + } + isc_throw(MemorySegmentGrown, "mapped memory segment grown, size: " + << impl_->base_sgmt_->get_size() << ", free size: " + << impl_->base_sgmt_->get_free_memory()); +} + +void +MemorySegmentMapped::deallocate(void* ptr, size_t /*size*/) { + impl_->base_sgmt_->deallocate(ptr); +} + +bool +MemorySegmentMapped::allMemoryDeallocated() const { + return (impl_->base_sgmt_->all_memory_deallocated()); +} + +size_t +MemorySegmentMapped::getSize() const { + return (impl_->base_sgmt_->get_size()); +} + +} // namespace util +} // namespace isc diff --git a/src/lib/util/memory_segment_mapped.h b/src/lib/util/memory_segment_mapped.h new file mode 100644 index 0000000000..24a8b897da --- /dev/null +++ b/src/lib/util/memory_segment_mapped.h @@ -0,0 +1,66 @@ +// Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC") +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#ifndef MEMORY_SEGMENT_MAPPED_H +#define MEMORY_SEGMENT_MAPPED_H + +#include + +#include + +#include + +namespace isc { +namespace util { + +/// \brief TBD +class MemorySegmentMapped : boost::noncopyable, public MemorySegment { +public: + /// arbitrary choice. + static const size_t INITIAL_SIZE = 32768; + + /// \brief Constructor in the read-only mode + /// + /// Creates a local memory segment object + MemorySegmentMapped(const std::string& filename); + + MemorySegmentMapped(const std::string& filename, bool create, + size_t initial_size = INITIAL_SIZE); + + /// \brief Destructor + virtual ~MemorySegmentMapped(); + + /// \brief Allocate/acquire a segment of memory. + virtual void* allocate(size_t size); + + /// \brief Deallocate/release a segment of memory. + virtual void deallocate(void* ptr, size_t size); + + virtual bool allMemoryDeallocated() const; + + size_t getSize() const; + +private: + struct Impl; + Impl* impl_; +}; + +} // namespace util +} // namespace isc + +#endif // MEMORY_SEGMENT_MAPPED_H + +// Local Variables: +// mode: c++ +// End: diff --git a/src/lib/util/tests/Makefile.am b/src/lib/util/tests/Makefile.am index 105322ff2f..30e0f3ea81 100644 --- a/src/lib/util/tests/Makefile.am +++ b/src/lib/util/tests/Makefile.am @@ -34,6 +34,7 @@ run_unittests_SOURCES += lru_list_unittest.cc run_unittests_SOURCES += interprocess_sync_file_unittest.cc run_unittests_SOURCES += interprocess_sync_null_unittest.cc run_unittests_SOURCES += memory_segment_local_unittest.cc +run_unittests_SOURCES += memory_segment_mapped_unittest.cc run_unittests_SOURCES += qid_gen_unittest.cc run_unittests_SOURCES += random_number_generator_unittest.cc run_unittests_SOURCES += sha1_unittest.cc diff --git a/src/lib/util/tests/memory_segment_mapped_unittest.cc b/src/lib/util/tests/memory_segment_mapped_unittest.cc new file mode 100644 index 0000000000..6e5ce0afac --- /dev/null +++ b/src/lib/util/tests/memory_segment_mapped_unittest.cc @@ -0,0 +1,219 @@ +// Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC") +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +#include + +#include + +#include +#include + +using namespace isc::util; +using boost::scoped_ptr; + +namespace { + +const char* const mapped_file = TEST_DATA_BUILDDIR "/test.mapped"; +const size_t DEFAULT_INITIAL_SIZE = 32 * 1024; // intentionally hardcoded + +class MemorySegmentMappedTest : public ::testing::Test { +protected: + MemorySegmentMappedTest() { + // Make sure the mapped file doesn't exist; actually it shouldn't + // exist in normal cases so remove() should normally fail (returning + // false), but we don't care. + boost::interprocess::file_mapping::remove(mapped_file); + + // Create a new segment with a new file. It also confirms the + // behavior of this mode of constructor. + segment_.reset(new MemorySegmentMapped(mapped_file, true)); + } + + ~MemorySegmentMappedTest() { + segment_.reset(); + boost::interprocess::file_mapping::remove(mapped_file); + } + + scoped_ptr segment_; +}; + +TEST_F(MemorySegmentMappedTest, createAndModify) { + // We are going to do the same set of basic tests twice; one after creating + // the mapped file, the other by re-opening the existing file in the + // read-write mode. + for (int i = 0; i < 2; ++i) { + // It should have the default size (intentionally hardcoded) + EXPECT_EQ(DEFAULT_INITIAL_SIZE, segment_->getSize()); + + // By default, nothing is allocated. + EXPECT_TRUE(segment_->allMemoryDeallocated()); + + void* ptr = segment_->allocate(1024); + EXPECT_NE(static_cast(0), ptr); + + // Now, we have an allocation: + EXPECT_FALSE(segment_->allMemoryDeallocated()); + + // deallocate it; it shouldn't cause disruption. + segment_->deallocate(ptr, 1024); + + EXPECT_TRUE(segment_->allMemoryDeallocated()); + + // re-open it. + segment_.reset(new MemorySegmentMapped(mapped_file, false)); + } +} + +TEST_F(MemorySegmentMappedTest, createWithSize) { + boost::interprocess::file_mapping::remove(mapped_file); + + // Re-create the mapped file with a non-default initial size, and confirm + // the size is actually the specified one. + segment_.reset(new MemorySegmentMapped(mapped_file, true, 64 * 1024)); + EXPECT_NE(DEFAULT_INITIAL_SIZE, 64 * 1024); + EXPECT_EQ(64 * 1024, segment_->getSize()); +} + +TEST_F(MemorySegmentMappedTest, openFail) { + // The given file is directory + EXPECT_THROW(MemorySegmentMapped("/", true), MemorySegmentOpenError); + + // file doesn't exist and directory isn't writable (we assume the root + // directory is not writable for the user running the test). + EXPECT_THROW(MemorySegmentMapped("/test.mapped", true), + MemorySegmentOpenError); + + // file doesn't exist and it's read-only (so open-only) + EXPECT_THROW(MemorySegmentMapped(TEST_DATA_BUILDDIR "/nosuchfile.mapped"), + MemorySegmentOpenError); + // Likewise. read-write mode but creation is suppressed. + EXPECT_THROW(MemorySegmentMapped(TEST_DATA_BUILDDIR "/nosuchfile.mapped", + false), + MemorySegmentOpenError); +} + +TEST_F(MemorySegmentMappedTest, allocate) { + // Various case of allocation. The simplest cases are covered above. + + // (Clearly) exceeding the available size, which should cause growing + // the segment + EXPECT_THROW(segment_->allocate(DEFAULT_INITIAL_SIZE + 1), + MemorySegmentGrown); + // The size should have been doubled. + EXPECT_EQ(DEFAULT_INITIAL_SIZE * 2,segment_->getSize()); + // In this case it should now succeed. + void* ptr = segment_->allocate(DEFAULT_INITIAL_SIZE + 1); + EXPECT_NE(static_cast(0), ptr); + + EXPECT_FALSE(segment_->allMemoryDeallocated()); + + // Same set of checks, but for a larger size. + EXPECT_THROW(segment_->allocate(DEFAULT_INITIAL_SIZE * 10), + MemorySegmentGrown); + ptr = segment_->allocate(DEFAULT_INITIAL_SIZE * 10); + EXPECT_NE(static_cast(0), ptr); +} + +TEST_F(MemorySegmentMappedTest, DISABLED_basics) { + MemorySegmentMapped segment(mapped_file); + + // By default, nothing is allocated. + EXPECT_TRUE(segment.allMemoryDeallocated()); + +#if 0 + void* ptr = segment->allocate(1024); + + // Now, we have an allocation: + EXPECT_FALSE(segment->allMemoryDeallocated()); + + void* ptr2 = segment->allocate(42); + + // Still: + EXPECT_FALSE(segment->allMemoryDeallocated()); + + // These should not fail, because the buffers have been allocated. + EXPECT_NO_FATAL_FAILURE(memset(ptr, 0, 1024)); + EXPECT_NO_FATAL_FAILURE(memset(ptr, 0, 42)); + + segment->deallocate(ptr, 1024); + + // Still: + EXPECT_FALSE(segment->allMemoryDeallocated()); + + segment->deallocate(ptr2, 42); + + // Now, we have an deallocated everything: + EXPECT_TRUE(segment->allMemoryDeallocated()); +#endif +} + +/* +TEST(MemorySegmentLocal, TestTooMuchMemory) { + auto_ptr segment(new MemorySegmentLocal()); + + EXPECT_THROW(segment->allocate(ULONG_MAX), bad_alloc); +} +*/ + +/* +TEST(MemorySegmentLocal, TestBadDeallocate) { + auto_ptr segment(new MemorySegmentLocal()); + + // By default, nothing is allocated. + EXPECT_TRUE(segment->allMemoryDeallocated()); + + void* ptr = segment->allocate(1024); + + // Now, we have an allocation: + EXPECT_FALSE(segment->allMemoryDeallocated()); + + // This should not throw + EXPECT_NO_THROW(segment->deallocate(ptr, 1024)); + + // Now, we have an deallocated everything: + EXPECT_TRUE(segment->allMemoryDeallocated()); + + ptr = segment->allocate(1024); + + // Now, we have another allocation: + EXPECT_FALSE(segment->allMemoryDeallocated()); + + // This should throw as the size passed to deallocate() is larger + // than what was allocated. + EXPECT_THROW(segment->deallocate(ptr, 2048), isc::OutOfRange); + + // This should not throw + EXPECT_NO_THROW(segment->deallocate(ptr, 1024)); + + // Now, we have an deallocated everything: + EXPECT_TRUE(segment->allMemoryDeallocated()); +} +*/ + +/* +TEST(MemorySegmentLocal, TestNullDeallocate) { + auto_ptr segment(new MemorySegmentLocal()); + + // By default, nothing is allocated. + EXPECT_TRUE(segment->allMemoryDeallocated()); + + // NULL deallocation is a no-op. + EXPECT_NO_THROW(segment->deallocate(NULL, 1024)); + + // This should still return true. + EXPECT_TRUE(segment->allMemoryDeallocated()); +} +*/ + +}