From: Thomas Markwalder Date: Fri, 12 Aug 2016 18:02:35 +0000 (-0400) Subject: [4294] Memfile and MySql now support recalulating IPv4 lease statistics X-Git-Tag: trac4631_base~6^2~17 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a8f85f25086f5cc9bc91a334fb788beadcf51a6a;p=thirdparty%2Fkea.git [4294] Memfile and MySql now support recalulating IPv4 lease statistics src/lib/dhcpsrv/cfg_subnets4.cc CfgSubnets4::removeStatistics() - added removal of all lease statistics per subnet, and global declined address stats CfgSubnets4::updateStatistics() - added call to LeaseMgr::recountAddressStats4 src/lib/dhcpsrv/lease.cc src/lib/dhcpsrv/lease.h Replaces lease state constants with LeaseState enumeration. src/lib/dhcpsrv/lease_mgr.cc src/lib/dhcpsrv/lease_mgr.h struct AddressStatsRow4 - contains the content of one row of the IPv4 lease statistical data result set class AddressStatsQuery4 - base class for constructing the IPv4 lease statistical data result set for an IPv4 lease storage LeaseMgr::recountAddressStats4() - new method which recalculates per-subnet and global stats for IPv4 leases LeaseMgr::startAddressStatsQuery4() - new virtual method that fetches the IPv4 lease statistical data result set src/lib/dhcpsrv/lease_mgr_factory.h src/lib/dhcpsrv/lease_mgr_factory.cc LeaseMgrFactory::haveInstance() - new static method which indicates whether or not the lease manager singleton exists src/lib/dhcpsrv/memfile_lease_mgr.h src/lib/dhcpsrv/memfile_lease_mgr.cc MemfileAddressStatsQuery4 - Derivation of AddressStatsQuery4, it constructs the IPv4 lease statistical data by iterating over IPv4 lease storage Memfile_LeaseMgr::startAddressStatsQuery4() - new virtual method which creates, starts, and returns a MemfileAddressStatsQuery4 src/lib/dhcpsrv/memfile_lease_storage.h Added an a per subnet_ID index to IPv4 storage src/lib/dhcpsrv/mysql_lease_mgr.h src/lib/dhcpsrv/mysql_lease_mgr.cc - Added RECOUNT_LEASE4_STATS query MySqlAddressStatsQuery4 Derivation of AddressStatsQuery4, it constructs the IPv4 lease statistical data by executing RECOUNT_LEASE4_STATS MySqlLeaseMgr::startAddressStatsQuery4() - new virtual method which creates, starts, and returns a MySqlAddressStatsQuery4 src/lib/dhcpsrv/tests/alloc_engine_utils.cc AllocEngine6Test::AllocEngine6Test() AllocEngine4Test::AllocEngine4Test() - moved lease mgr create up above configuration commit src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc ~CfgMySQLDbAccessTest() - added destruction of lease manager singleton, otherwise subsequent tests can fail src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc GenericLeaseMgrTest::checkStat() - new method for comparing a stat GenericLeaseMgrTest::checkAddressStats4() - new method for comparing a list of stats GenericLeaseMgrTest::makeLease4() - new method for making a minimal lease GenericLeaseMgrTest::testRecountAddressStats4() - new method which tests a lease manager's ability to recalculate the IPv4 lease statistics src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc TEST_F(MemfileLeaseMgrTest, recountAddressStats4) - new test which tests Memfile_LeaseMgr's ability to recalculate IPv4 lease statistics src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc TEST_F(MySqlLeaseMgrTest, recountAddressStats4) - new test which tests MySqlLeaseMgr's ability to recalculate IPv4 lease statistics --- diff --git a/src/lib/dhcpsrv/cfg_subnets4.cc b/src/lib/dhcpsrv/cfg_subnets4.cc index 6c6fd408d3..43b1883053 100644 --- a/src/lib/dhcpsrv/cfg_subnets4.cc +++ b/src/lib/dhcpsrv/cfg_subnets4.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -232,18 +233,21 @@ CfgSubnets4::removeStatistics() { using namespace isc::stats; // For each v4 subnet currently configured, remove the statistic. - /// @todo: May move this to CfgSubnets4 class if there will be more - /// statistics here. + StatsMgr& stats_mgr = StatsMgr::instance(); for (Subnet4Collection::const_iterator subnet4 = subnets_.begin(); subnet4 != subnets_.end(); ++subnet4) { + SubnetID subnet_id = (*subnet4)->getID(); + stats_mgr.del(StatsMgr::generateName("subnet", subnet_id, + "total-addresses")); - StatsMgr::instance().del(StatsMgr::generateName("subnet", - (*subnet4)->getID(), - "total-addresses")); + stats_mgr.del(StatsMgr::generateName("subnet", subnet_id, + "assigned-addresses")); - StatsMgr::instance().del(StatsMgr::generateName("subnet", - (*subnet4)->getID(), - "assigned-addresses")); + stats_mgr.del(StatsMgr::generateName("subnet", subnet_id, + "declined-addresses")); + + stats_mgr.del(StatsMgr::generateName("subnet", subnet_id, + "declined-reclaimed-addresses")); } } @@ -251,14 +255,21 @@ void CfgSubnets4::updateStatistics() { using namespace isc::stats; - /// @todo: May move this to CfgSubnets4 class if there will be more - /// statistics here. - for (Subnet4Collection::const_iterator subnet = subnets_.begin(); - subnet != subnets_.end(); ++subnet) { + StatsMgr& stats_mgr = StatsMgr::instance(); + for (Subnet4Collection::const_iterator subnet4 = subnets_.begin(); + subnet4 != subnets_.end(); ++subnet4) { + SubnetID subnet_id = (*subnet4)->getID(); + + stats_mgr.setValue(StatsMgr:: + generateName("subnet", subnet_id, "total-addresses"), + static_cast + ((*subnet4)->getPoolCapacity(Lease:: + TYPE_V4))); + } - StatsMgr::instance().setValue( - StatsMgr::generateName("subnet", (*subnet)->getID(), "total-addresses"), - static_cast((*subnet)->getPoolCapacity(Lease::TYPE_V4))); + // If we have subnets and a lease mgr, recount the least statistics + if (subnets_.begin() != subnets_.end() && LeaseMgrFactory::haveInstance()) { + LeaseMgrFactory::instance().recountAddressStats4(); } } diff --git a/src/lib/dhcpsrv/dhcpsrv_messages.mes b/src/lib/dhcpsrv/dhcpsrv_messages.mes index 714148548c..db03ed0bb0 100644 --- a/src/lib/dhcpsrv/dhcpsrv_messages.mes +++ b/src/lib/dhcpsrv/dhcpsrv_messages.mes @@ -923,3 +923,6 @@ lease from the Cassandra database for the specified address. % DHCPSRV_CQL_UPDATE_ADDR6 updating IPv6 lease for address %1 A debug message issued when the server is attempting to update IPv6 lease from the Cassandra database for the specified address. + +% TOMS_UTILITY_MESSAGE %1 +Handy log message that should be deleted diff --git a/src/lib/dhcpsrv/lease.cc b/src/lib/dhcpsrv/lease.cc index 5854f06900..d8d15613d1 100644 --- a/src/lib/dhcpsrv/lease.cc +++ b/src/lib/dhcpsrv/lease.cc @@ -15,10 +15,6 @@ using namespace std; namespace isc { namespace dhcp { -const uint32_t Lease::STATE_DEFAULT = 0x0; -const uint32_t Lease::STATE_DECLINED = 0x1; -const uint32_t Lease::STATE_EXPIRED_RECLAIMED = 0x2; - Lease::Lease(const isc::asiolink::IOAddress& addr, uint32_t t1, uint32_t t2, uint32_t valid_lft, SubnetID subnet_id, time_t cltt, const bool fqdn_fwd, const bool fqdn_rev, diff --git a/src/lib/dhcpsrv/lease.h b/src/lib/dhcpsrv/lease.h index d1ab300f11..365cb06f53 100644 --- a/src/lib/dhcpsrv/lease.h +++ b/src/lib/dhcpsrv/lease.h @@ -40,18 +40,18 @@ struct Lease { /// @return text decription static std::string typeToText(Type type); - /// @name Common lease states constants. + /// @name Enumeration of lease states //@{ - /// - /// @brief A lease in the default state. - static const uint32_t STATE_DEFAULT; - - /// @brief Declined lease. - static const uint32_t STATE_DECLINED; - - /// @brief Expired and reclaimed lease. - static const uint32_t STATE_EXPIRED_RECLAIMED; - + typedef enum { + /// @brief A lease in the default (assigned) state. + STATE_DEFAULT, + /// @brief Declined lease. + STATE_DECLINED, + /// @brief Expired and reclaimed lease. + STATE_EXPIRED_RECLAIMED, + /// @brief The number of defined lease states. + NUM_LEASE_STATES + } LeaseState; //@} /// @brief Returns name(s) of the basic lease state(s). diff --git a/src/lib/dhcpsrv/lease_mgr.cc b/src/lib/dhcpsrv/lease_mgr.cc index 58f95d7dc9..491c5a0010 100644 --- a/src/lib/dhcpsrv/lease_mgr.cc +++ b/src/lib/dhcpsrv/lease_mgr.cc @@ -6,8 +6,11 @@ #include +#include +#include #include #include +#include #include #include @@ -44,6 +47,82 @@ LeaseMgr::getLease6(Lease::Type type, const DUID& duid, return (*col.begin()); } +void +LeaseMgr::recountAddressStats4() { + using namespace stats; + + StatsMgr& stats_mgr = StatsMgr::instance(); + + AddressStatsQuery4Ptr query = startAddressStatsQuery4(); + if (!query) { + /// NULL means not backend does not support recounting. + return; + } + + // Zero out the global stats. (Ok, so currently there's only one + // that should be cleared. "reclaimed-declined-addresses" never + // gets zeroed. @todo discuss with Tomek the rational of not + // clearing it when we clear the rest. + int64_t zero = 0; + stats_mgr.setValue("declined-addresses", zero); + stats_mgr.setValue("declined-reclaimed-addresses", zero); + + // Clear subnet level stats. This ensures we don't end up with corner + // cases that leave stale values in place. + const Subnet4Collection* subnets = + CfgMgr::instance().getCurrentCfg()->getCfgSubnets4()->getAll(); + + for (Subnet4Collection::const_iterator subnet = subnets->begin(); + subnet != subnets->end(); ++subnet) { + SubnetID subnet_id = (*subnet)->getID(); + stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id, + "assigned-addresses"), + zero); + + stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id, + "declined-addresses"), + zero); + stats_mgr.setValue(StatsMgr::generateName("subnet", subnet_id, + "declined-reclaimed-addresses"), + zero); + } + + // Get counts per state per subnet. Iterate over the result set + // updating the subnet and global values. + AddressStatsRow4 row; + while (query->getNextRow(row)) { + switch(row.lease_state_) { + case Lease::STATE_DEFAULT: + // Set subnet level value. + stats_mgr.setValue(StatsMgr::generateName("subnet", + row.subnet_id_, + "assigned-addresses"), + row.state_count_); + break; + + case Lease::STATE_DECLINED: + // Set subnet level value. + stats_mgr.setValue(StatsMgr::generateName("subnet", + row.subnet_id_, + "declined-addresses"), + row.state_count_); + + // Add to the global value. + stats_mgr.addValue("declined-addresses", row.state_count_); + break; + + default: + // Not one we're tracking. + break; + } + } +} + +AddressStatsQuery4Ptr +LeaseMgr::startAddressStatsQuery4() { + return(AddressStatsQuery4Ptr()); +} + std::string LeaseMgr::getDBVersion() { isc_throw(NotImplemented, "LeaseMgr::getDBVersion() called"); diff --git a/src/lib/dhcpsrv/lease_mgr.h b/src/lib/dhcpsrv/lease_mgr.h index c31f2dfa12..e8123e840f 100644 --- a/src/lib/dhcpsrv/lease_mgr.h +++ b/src/lib/dhcpsrv/lease_mgr.h @@ -146,6 +146,70 @@ public: virtual ~SqlExchange() {}; ExchangeColumnInfoContainer parameters_; ///< Column names and types }; + +/// @brief Contains a single row of IPv4 lease statistical data +/// +/// The contents of the row consist of a subnet ID, a lease state, +/// and the number of leases in that state for that subnet ID. +struct AddressStatsRow4 { + /// @brief Default constructor + AddressStatsRow4() : + subnet_id_(0), lease_state_(Lease::STATE_DEFAULT), state_count_(0) { + } + + /// @brief Constructor + /// + /// @param subnet_id The subnet id to which this data applies + /// @param lease_state The lease state counted + /// @param state_count The count of leases in the lease state + AddressStatsRow4(const SubnetID& subnet_id, + const Lease::LeaseState& lease_state, + const int64_t state_count) + : subnet_id_(subnet_id), lease_state_(lease_state), + state_count_(state_count) { + } + + /// @brief The subnet ID to which this data applies + SubnetID subnet_id_; + /// @brief The lease_state to which the count applies + uint32_t lease_state_; + /// @brief state_count The count of leases in the lease state + int64_t state_count_; +}; + +/// @brief Base class for fulfilling IPv4 statistical lease data query +/// +/// LeaseMgr derivations implement this class such that it provides +/// upto date IPv4 statistical lease data organized as rows of +/// AddressStatsRow4 instances. The rows must be accessible in +/// ascending order by subnet id. +class AddressStatsQuery4 { +public: + /// @brief Default constructor + AddressStatsQuery4() {}; + + /// @brief virtual destructor + virtual ~AddressStatsQuery4() {}; + + /// @brief Executes the query + /// + /// This method should conduct whatever steps are required to + /// calculate the IPv4 lease statistical data by examining the + /// IPv4 lease data and making that results available row by row. + virtual void start() {}; + + /// @brief Fetches the next row of data + /// + /// @param[out] row Storage into which the row is fetched + /// + /// @return True if a row was fetched, false if there are no + /// more rows. + virtual bool getNextRow(AddressStatsRow4& row) { return(false); }; +}; + +/// @brief Defines a pointer to an AddressStatsQuery4. +typedef boost::shared_ptr AddressStatsQuery4Ptr; + /// @brief Abstract Lease Manager /// /// This is an abstract API for lease database backends. It provides unified @@ -397,6 +461,36 @@ public: /// @return Number of leases deleted. virtual uint64_t deleteExpiredReclaimedLeases6(const uint32_t secs) = 0; + /// @brief Recalculates per-subnet and global stats for IPv4 leases + /// + /// This method recalculates the following statistics: + /// per-subnet: + /// - assigned-addresses + /// - declined-addresses + /// - declined-reclaimed-addresses (reset to zero) + /// global: + /// - declined-addresses + /// - declined-reclaimed-addresses (reset to zero) + /// + /// It invokes the virtual method, startAddressStatsQuery4(), which + /// returns an instance of an AddressStats4Qry. The query + /// query contains a "result set" where each row is an AddressStatRow4 + /// that contains a subnet id, a lease state, the number of leases in that + /// state and is ordered by subnet id. The method iterates over the + /// result set rows, setting the appropriate statistic per subnet and + /// adding to the approporate global statistic. + void recountAddressStats4(); + + /// @brief Virtual method which creates and runs the IPv4 lease stats query + /// + /// LeaseMgr derivations implement this method such that it creates and + /// returns an instance of an AddressStatsQuery whose result set has been + /// populated with upto date IPv4 lease statistical data. Each row of the + /// result set is an AddressStatRow4 which ordered ascending by subnet ID. + /// + /// @return A populated AddressStatsQuery4 + virtual AddressStatsQuery4Ptr startAddressStatsQuery4(); + /// @brief Return backend type /// /// Returns the type of the backend (e.g. "mysql", "memfile" etc.) diff --git a/src/lib/dhcpsrv/lease_mgr_factory.cc b/src/lib/dhcpsrv/lease_mgr_factory.cc index 5f2f33a4f3..a9f1507b2b 100644 --- a/src/lib/dhcpsrv/lease_mgr_factory.cc +++ b/src/lib/dhcpsrv/lease_mgr_factory.cc @@ -101,6 +101,11 @@ LeaseMgrFactory::destroy() { getLeaseMgrPtr().reset(); } +bool +LeaseMgrFactory::haveInstance() { + return (getLeaseMgrPtr().get()); +} + LeaseMgr& LeaseMgrFactory::instance() { LeaseMgr* lmptr = getLeaseMgrPtr().get(); diff --git a/src/lib/dhcpsrv/lease_mgr_factory.h b/src/lib/dhcpsrv/lease_mgr_factory.h index 279058bed8..4c6baf1270 100644 --- a/src/lib/dhcpsrv/lease_mgr_factory.h +++ b/src/lib/dhcpsrv/lease_mgr_factory.h @@ -85,7 +85,10 @@ public: /// create() to create one before calling this method. static LeaseMgr& instance(); - + /// @brief Indicates if the lease manager has been instantiated. + /// + /// @return True if the lease manager instance exists, false otherwise. + static bool haveInstance(); private: /// @brief Hold pointer to lease manager diff --git a/src/lib/dhcpsrv/memfile_lease_mgr.cc b/src/lib/dhcpsrv/memfile_lease_mgr.cc index 51314b4cd8..1ad41cd1c2 100644 --- a/src/lib/dhcpsrv/memfile_lease_mgr.cc +++ b/src/lib/dhcpsrv/memfile_lease_mgr.cc @@ -256,6 +256,143 @@ LFCSetup::getExitStatus() const { return (process_->getExitStatus(pid_)); } + +/// @brief Memfile derivation of the IPv4 statistical lease data query +/// +/// This class is used to recalculate IPv4 lease statistics for Memfile +/// lease storage. It does so by iterating over the given storage, +/// accumulating counts of leases in each of the monitored lease states +/// for each subnet and storing these counts in an internal collection. +/// The populated result set will contain one entry per monitored state +/// per subnet. +/// +class MemfileAddressStatsQuery4 : public AddressStatsQuery4 { +public: + /// @brief Constructor + /// + /// @param storage4 A pointer to the v4 lease storage to be counted + MemfileAddressStatsQuery4(Lease4Storage& storage4); + + /// @brief Destructor + virtual ~MemfileAddressStatsQuery4() {}; + + /// @brief Creates the IPv4 lease statistical data result set + /// + /// The result is populated by iterating over the IPv4 leases in storage, + /// in ascending order by subnet ID, accumulating the lease state counts. + /// At the completion of all entries for a given subnet, the counts are + /// used to create AddressStatsRow4 instances which are appended to an + /// internal vector. The process results in a vector containing one entry + /// per state per subnet. + /// + /// Currently the states counted are: + /// + /// - Lease::STATE_DEFAULT (i.e. assigned) + /// - Lease::STATE_DECLINED + virtual void start(); + + /// @brief Fetches the next row in the result set + /// + /// Once the internal result set has been populated by invoking the + /// the start() method, this method is used to iterate over the + /// result set rows. Once the last row has been fetched, subsequent + /// calls will return false. + /// @param row Storage for the fetched row + /// + /// @return True if the fetch succeeded, false if there are no more + /// rows to fetch. + virtual bool getNextRow(AddressStatsRow4& row); + + /// @brief Returns the number of rows in the result set + /// @todo, should this be a virtual member of the base class? + int getRowCount(); + +private: + /// @brief The Memfile storage containing the IPv4 leases to analyze + Lease4Storage& storage4_; + + /// @brief A vector containing the "result set" + std::vector rows_; + + /// @brief An iterator for accessing the next row within the result set + std::vector::iterator next_pos_; +}; + +MemfileAddressStatsQuery4::MemfileAddressStatsQuery4(Lease4Storage& storage4) + : storage4_(storage4), rows_(0), next_pos_(rows_.end()) {}; + +void +MemfileAddressStatsQuery4::start() { + // Get the subnet_id index + const Lease4StorageSubnetIdIndex& idx = storage4_.get(); + + // Iterate over the leases in order by subnet, accumulating per + // subnet counts for each state of interest. As we finish each + // subnet, add the appropriate rows to our result set. + SubnetID cur_id = 0; + int64_t assigned = 0; + int64_t declined = 0; + for(Lease4StorageSubnetIdIndex::const_iterator lease = idx.begin(); + lease != idx.end(); ++lease) { + + // If we've hit the next subnet, add rows for the current subnet + // and wipe the accumulators + if ((*lease)->subnet_id_ > cur_id) { + if (cur_id > 0) { + rows_.push_back(AddressStatsRow4(cur_id,Lease::STATE_DEFAULT, + assigned)); + assigned = 0; + rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, + declined)); + declined = 0; + } + + // Update current subnet id + cur_id = (*lease)->subnet_id_; + } + + // Bump the appropriate accumulator + switch ((*lease)->state_) { + case Lease::STATE_DEFAULT: + ++assigned; + break; + case Lease::STATE_DECLINED: + ++declined; + break; + default: + // Not one we're tracking. + break; + } + } + + // Make the rows for last subnet, unless there were no rows + if (idx.begin() != idx.end()) { + rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DEFAULT, + assigned)); + rows_.push_back(AddressStatsRow4(cur_id, Lease::STATE_DECLINED, + declined)); + } + + // Set the next row position to the beginning of the rows. + next_pos_ = rows_.begin(); +} + +bool +MemfileAddressStatsQuery4::getNextRow(AddressStatsRow4& row) { + if (next_pos_ == rows_.end()) { + return (false); + } + + row = *next_pos_; + ++next_pos_; + return (true); +} + +int +MemfileAddressStatsQuery4::getRowCount() { + return (rows_.size()); +} + // Explicit definition of class static constants. Values are given in the // declaration so they're not needed here. const int Memfile_LeaseMgr::MAJOR_VERSION; @@ -299,6 +436,7 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& param } lfcSetup(conversion_needed); } + } Memfile_LeaseMgr::~Memfile_LeaseMgr() { @@ -1048,5 +1186,12 @@ void Memfile_LeaseMgr::lfcExecute(boost::shared_ptr& lease_file) } } +AddressStatsQuery4Ptr +Memfile_LeaseMgr::startAddressStatsQuery4() { + AddressStatsQuery4Ptr query(new MemfileAddressStatsQuery4(storage4_)); + query->start(); + return(query); +} + } // end of namespace isc::dhcp } // end of namespace isc diff --git a/src/lib/dhcpsrv/memfile_lease_mgr.h b/src/lib/dhcpsrv/memfile_lease_mgr.h index 1ace364cf7..8bc4c72b85 100644 --- a/src/lib/dhcpsrv/memfile_lease_mgr.h +++ b/src/lib/dhcpsrv/memfile_lease_mgr.h @@ -92,6 +92,7 @@ public: /// @} + /// @brief Specifies universe (V4, V6) /// /// This enumeration is used by various functions in Memfile %Lease Manager, @@ -594,6 +595,14 @@ public: int getLFCExitStatus() const; //@} + /// @brief Creates and runs the IPv4 lease stats query + /// + /// It creates an instance of a MemfileAddressStatsQuery4 and then + /// invokes it's start method in which the query constructs its + /// statistical data result set. The query object is then returned. + /// + /// @return The populated query as a pointer to an AddressStatsQuery4 + virtual AddressStatsQuery4Ptr startAddressStatsQuery4(); /// @name Protected methods used for %Lease File Cleanup. /// The following methods are protected so as they can be accessed and diff --git a/src/lib/dhcpsrv/memfile_lease_storage.h b/src/lib/dhcpsrv/memfile_lease_storage.h index 566a74d7c5..fba499672f 100644 --- a/src/lib/dhcpsrv/memfile_lease_storage.h +++ b/src/lib/dhcpsrv/memfile_lease_storage.h @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -27,6 +27,9 @@ namespace dhcp { /// @brief Tag for indexes by address. struct AddressIndexTag { }; +/// @brief Tag for indexes by subnet id. +struct SubnetIdIndexTag { }; + /// @brief Tag for indexes by DUID, IAID, lease type tuple. struct DuidIaidTypeIndexTag { }; @@ -135,6 +138,15 @@ typedef boost::multi_index_container< boost::multi_index::member >, + boost::multi_index::ordered_non_unique< + boost::multi_index::tag, + // The subnet id is held in the subnet_id_ member of Lease4 + // class. Note that the subnet_id_ is defined in the base + // class (Lease) so we have to point to this class rather + // than derived class: Lease4. + boost::multi_index::member + >, + // Specification of the second index starts here. boost::multi_index::ordered_non_unique< boost::multi_index::tag, @@ -232,6 +244,9 @@ typedef Lease6Storage::index::type Lease6StorageExpirationIn /// @brief DHCPv4 lease storage index by address. typedef Lease4Storage::index::type Lease4StorageAddressIndex; +/// @brief DHCPv4 lease storage index by subnet id. +typedef Lease4Storage::index::type Lease4StorageSubnetIdIndex; + /// @brief DHCPv4 lease storage index by exiration time. typedef Lease4Storage::index::type Lease4StorageExpirationIndex; diff --git a/src/lib/dhcpsrv/mysql_lease_mgr.cc b/src/lib/dhcpsrv/mysql_lease_mgr.cc index 410880af36..87925d1f52 100644 --- a/src/lib/dhcpsrv/mysql_lease_mgr.cc +++ b/src/lib/dhcpsrv/mysql_lease_mgr.cc @@ -205,6 +205,9 @@ TaggedStatement tagged_statements[] = { "hostname = ?, hwaddr = ?, hwtype = ?, hwaddr_source = ?, " "state = ? " "WHERE address = ?"}, + {MySqlLeaseMgr::RECOUNT_LEASE4_STATS, + "SELECT subnet_id, state, count(state) as state_count " + "FROM lease4 group by subnet_id, state"}, // End of list sentinel {MySqlLeaseMgr::NUM_STATEMENTS, NULL} }; @@ -1214,6 +1217,143 @@ private: uint32_t state_; ///< Lease state. }; +/// @brief MySql derivation of the IPv4 statistical lease data query +/// +/// This class is used to recalculate IPv4 lease statistics for MySQL +/// lease storage. It does so by executing a query which returns a result +/// containining contain one row per monitored state per subnet, ordered +/// by subnet id in ascending order. +/// +class MySqlAddressStatsQuery4 : public AddressStatsQuery4 { +public: + /// @brief Constructor + /// + /// @param conn A open connection to the database housing the lease data + MySqlAddressStatsQuery4(MySqlConnection& conn); + + /// @brief Destructor + virtual ~MySqlAddressStatsQuery4(); + + /// @brief Creates the IPv4 lease statistical data result set + /// + /// The result set is populated by executing an SQL query against the + /// lease4 table which sums the leases per lease state per subnet id. + /// The query used is the prepared statement identified by + /// MySqlLeaseMgr::RECOUNT_LEASE4_STATS. This method creates the binds + /// the statement to the output bind array and then executes the + /// statement. + void start(); + + /// @brief Fetches the next row in the result set + /// + /// Once the internal result set has been populated by invoking the + /// the start() method, this method is used to iterate over the + /// result set rows. Once the last row has been fetched, subsequent + /// calls will return false. + /// + /// @param row Storage for the fetched row + /// + /// @return True if the fetch succeeded, false if there are no more + /// rows to fetch. + bool getNextRow(AddressStatsRow4& row); + +private: + + /// @brief Analyzes the given statement outcome status + /// + /// Wrapper method around the MySqlConnection:checkError() that is + /// used to generate the appropriate exception if the status indicates + /// an error. + //// + /// a DbOperation error + /// @param status The MySQL statement execution outcome status + /// @param what invocation context message which will be included in + /// any exception + void checkError(int status, const char* what) const; + + /// @brief Database connection to use to execute the query + MySqlConnection& conn_; + + /// @brief The query's prepared statement + MYSQL_STMT *statement_; + + /// @brief Bind array used to store the query result set; + std::vector bind_; + + /// @brief Member struct that is bound to the statement; + AddressStatsRow4 stat_row_; +}; + +MySqlAddressStatsQuery4::MySqlAddressStatsQuery4(MySqlConnection& conn) + : conn_(conn), statement_(conn_.statements_[MySqlLeaseMgr + ::RECOUNT_LEASE4_STATS]), + bind_(3) { +} + +MySqlAddressStatsQuery4::~MySqlAddressStatsQuery4() { + (void) mysql_stmt_free_result(statement_); +} + + +void +MySqlAddressStatsQuery4::start() { + // subnet_id: unsigned int + bind_[0].buffer_type = MYSQL_TYPE_LONG; + bind_[0].buffer = reinterpret_cast(&stat_row_.subnet_id_); + bind_[0].is_unsigned = MLM_TRUE; + + // state: uint32_t + bind_[1].buffer_type = MYSQL_TYPE_LONG; + bind_[1].buffer = reinterpret_cast(&stat_row_.lease_state_); + bind_[1].is_unsigned = MLM_TRUE; + + // state_count_: uint32_t + bind_[2].buffer_type = MYSQL_TYPE_LONG; + bind_[2].buffer = reinterpret_cast(&stat_row_.state_count_); + bind_[2].is_unsigned = MLM_TRUE; + + // Set up the MYSQL_BIND array for the data being returned + // and bind it to the statement. + int status = mysql_stmt_bind_result(statement_, &bind_[0]); + checkError(status, "RECOUNT_LEASE4_STATS: outbound binding failed"); + + // Execute the statement + status = mysql_stmt_execute(statement_); + checkError(status, "RECOUNT_LEASE4_STATS: unable to execute"); + + // Ensure that all the lease information is retrieved in one go to avoid + // overhead of going back and forth between client and server. + status = mysql_stmt_store_result(statement_); + checkError(status, "RECOUNT_LEASE4_STATS: results storage setup failed"); +} + +bool +MySqlAddressStatsQuery4::getNextRow(AddressStatsRow4& row) { + bool have_row = false; + int status = mysql_stmt_fetch(statement_); + if (status == MLM_MYSQL_FETCH_SUCCESS) { + row = stat_row_; + have_row = true; + } else if (status != MYSQL_NO_DATA) { + checkError(status, "RECOUNT_LEASE4_STATS: getNextRow failed"); + } + + return (have_row); +} + +void +MySqlAddressStatsQuery4::checkError(int status, const char* what) const { + conn_.checkError(status, MySqlLeaseMgr::RECOUNT_LEASE4_STATS, what); +} + +AddressStatsQuery4Ptr +MySqlLeaseMgr::startAddressStatsQuery4() { + AddressStatsQuery4Ptr query(new MySqlAddressStatsQuery4(conn_)); + query->start(); + return(query); +} + + @@ -2025,7 +2165,6 @@ MySqlLeaseMgr::getVersion() const { return (std::make_pair(major, minor)); } - void MySqlLeaseMgr::commit() { LOG_DEBUG(dhcpsrv_logger, DHCPSRV_DBG_TRACE_DETAIL, DHCPSRV_MYSQL_COMMIT); diff --git a/src/lib/dhcpsrv/mysql_lease_mgr.h b/src/lib/dhcpsrv/mysql_lease_mgr.h index a5824becbb..b65c52a31b 100644 --- a/src/lib/dhcpsrv/mysql_lease_mgr.h +++ b/src/lib/dhcpsrv/mysql_lease_mgr.h @@ -25,7 +25,6 @@ namespace dhcp { class MySqlLease4Exchange; class MySqlLease6Exchange; - /// @brief MySQL Lease Manager /// /// This class provides the \ref isc::dhcp::LeaseMgr interface to the MySQL @@ -410,6 +409,7 @@ public: INSERT_LEASE6, // Add entry to lease6 table UPDATE_LEASE4, // Update a Lease4 entry UPDATE_LEASE6, // Update a Lease6 entry + RECOUNT_LEASE4_STATS, // Fetches address statisics NUM_STATEMENTS // Number of statements }; @@ -590,6 +590,15 @@ private: uint64_t deleteExpiredReclaimedLeasesCommon(const uint32_t secs, StatementIndex statement_index); + /// @brief Creates and runs the IPv4 lease stats query + /// + /// It creates an instance of a MySqlAddressStatsQuery4 and then + /// invokes its start method, which fetches its statistical data + /// result set by executing the RECOUNT_LEASE_STATS4 query. + /// The query object is then returned. + /// + /// @return The populated query as a pointer to an AddressStatsQuery4 + virtual AddressStatsQuery4Ptr startAddressStatsQuery4(); /// @brief Check Error and Throw Exception /// diff --git a/src/lib/dhcpsrv/tests/alloc_engine_utils.cc b/src/lib/dhcpsrv/tests/alloc_engine_utils.cc index 8f86f6a5e9..9beb306609 100644 --- a/src/lib/dhcpsrv/tests/alloc_engine_utils.cc +++ b/src/lib/dhcpsrv/tests/alloc_engine_utils.cc @@ -123,6 +123,8 @@ AllocEngine4Test::generateDeclinedLease(const std::string& addr, AllocEngine6Test::AllocEngine6Test() { CfgMgr::instance().clear(); + factory_.create("type=memfile universe=6 persist=false"); + duid_ = DuidPtr(new DUID(std::vector(8, 0x42))); iaid_ = 42; @@ -141,7 +143,6 @@ AllocEngine6Test::AllocEngine6Test() { initFqdn("", false, false); - factory_.create("type=memfile universe=6 persist=false"); } void @@ -525,6 +526,9 @@ AllocEngine4Test::initSubnet(const asiolink::IOAddress& pool_start, } AllocEngine4Test::AllocEngine4Test() { + + factory_.create("type=memfile universe=4 persist=false"); + // Create fresh instance of the HostMgr, and drop any previous HostMgr state. HostMgr::instance().create(); @@ -548,7 +552,6 @@ AllocEngine4Test::AllocEngine4Test() { initSubnet(IOAddress("192.0.2.100"), IOAddress("192.0.2.109")); cfg_mgr.commit(); - factory_.create("type=memfile universe=4 persist=false"); // Create a default context. Note that remaining parameters must be // assigned when needed. diff --git a/src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc b/src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc index 49d3a378ff..25e7c053c3 100644 --- a/src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc +++ b/src/lib/dhcpsrv/tests/cfg_db_access_unittest.cc @@ -88,6 +88,7 @@ public: /// @brief Destructor. virtual ~CfgMySQLDbAccessTest() { destroyMySQLSchema(); + LeaseMgrFactory::destroy(); } }; diff --git a/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc b/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc index a94a3a2b5c..d8ced0e5f2 100644 --- a/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc +++ b/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc @@ -5,11 +5,18 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. #include + +#include +#include +#include #include #include -#include -#include +#include + +#include + #include + #include using namespace std; @@ -57,6 +64,7 @@ GenericLeaseMgrTest::GenericLeaseMgrTest() /// a template leasetype6_.push_back(LEASETYPE6[i]); } + } GenericLeaseMgrTest::~GenericLeaseMgrTest() { @@ -2381,6 +2389,169 @@ GenericLeaseMgrTest::testGetDeclinedLeases6() { } } +void +GenericLeaseMgrTest::checkStat(const std::string& name, + const int64_t expected_value) { + stats::ObservationPtr obs = + stats::StatsMgr::instance().getObservation(name); + + ASSERT_TRUE(obs) << " stat: " << name << " not found "; + ASSERT_EQ(expected_value, obs->getInteger().first) + << " stat: " << name << " value wrong"; +} + +void +GenericLeaseMgrTest::checkAddressStats4(const StatValMapList& expectedStats) { + // Global accumulators + int64_t declined_addresses = 0; + int64_t declined_reclaimed_addresses = 0; + +#if 0 + isc::data::ConstElementPtr allstats = stats::StatsMgr::instance().getAll(); + std::cout << "ALL: "; + allstats->toJSON(std::cout); + std::cout << std::endl; +#endif + + // Iterate over all stats for each subnet + for (int subnet_idx = 0; subnet_idx < expectedStats.size(); ++subnet_idx) { + BOOST_FOREACH(StatValPair expectedStat, expectedStats[subnet_idx]) { + // Verify the per subnet value. + checkStat(stats::StatsMgr::generateName("subnet", subnet_idx+1, + expectedStat.first), + expectedStat.second); + + // Add the value to globals as needed. + if (expectedStat.first == "declined-addresses") { + declined_addresses += expectedStat.second; + } else if (expectedStat.first == "declined-reclaimed-addresses") { + declined_reclaimed_addresses += expectedStat.second; + } + } + } + + // Verify the globals. + checkStat("declined-addresses", declined_addresses); + checkStat("declined-reclaimed-addresses", declined_reclaimed_addresses); +} + +void +GenericLeaseMgrTest::makeLease4(const std::string& address, + const SubnetID& subnet_id, + const Lease::LeaseState& state) { + Lease4Ptr lease(new Lease4()); + + // set the address + lease->addr_ = IOAddress(address); + + // make a MAC from the address + std::vector hwaddr = lease->addr_.toBytes(); + hwaddr.push_back(0); + hwaddr.push_back(0); + + lease->hwaddr_.reset(new HWAddr(hwaddr, HTYPE_ETHER)); + lease->valid_lft_ = 86400; + lease->cltt_ = 168256; + lease->subnet_id_ = subnet_id; + lease->state_ = state; + ASSERT_TRUE(lmptr_->addLease(lease)); +} + +void +GenericLeaseMgrTest::testRecountAddressStats4() { + using namespace stats; + + StatsMgr::instance().removeAll(); + + // create subnets + CfgSubnets4Ptr cfg = + CfgMgr::instance().getStagingCfg()->getCfgSubnets4(); + + // Create 3 subnets. + Subnet4Ptr subnet; + Pool4Ptr pool; + + subnet.reset(new Subnet4(IOAddress("192.0.1.0"), 24, 1, 2, 3, 1)); + pool.reset(new Pool4(IOAddress("192.0.1.0"), 24)); + subnet->addPool(pool); + cfg->add(subnet); + + subnet.reset(new Subnet4(IOAddress("192.0.2.0"), 24, 1, 2, 3, 2)); + pool.reset(new Pool4(IOAddress("192.0.2.0"), 24)); + subnet->addPool(pool); + cfg->add(subnet); + + int num_subnets = 2; + + ASSERT_NO_THROW(CfgMgr::instance().commit()); + + // Create the expected stats list. At this point, the only stat + // that should be non-zero is total-addresses. + StatValMapList expectedStats(num_subnets); + for (int i = 0; i < num_subnets; ++i) { + expectedStats[i]["total-addresses"] = 256; + expectedStats[i]["assigned-addresses"] = 0; + expectedStats[i]["declined-addresses"] = 0; + expectedStats[i]["declined-reclaimed-addresses"] = 0; + } + + // Make sure stats are as expected. + ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats)); + + // Recount stats. We should have the same results. + ASSERT_NO_THROW(lmptr_->recountAddressStats4()); + + // Make sure stats are as expected. + ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats)); + + // Now let's insert some leases into subnet 1. + int subnet_id = 1; + + // Insert one lease in default state, i.e. assigned. + makeLease4("192.0.1.1", subnet_id); + + // Insert one lease in declined state. + makeLease4("192.0.1.2", subnet_id, Lease::STATE_DECLINED); + + // Insert one lease in the expired state. + makeLease4("192.0.1.3", subnet_id, Lease::STATE_EXPIRED_RECLAIMED); + + // Insert another lease in default state, i.e. assigned. + makeLease4("192.0.1.4", subnet_id); + + // Update the expected stats list for subnet 1. + expectedStats[subnet_id - 1]["assigned-addresses"] = 2; + expectedStats[subnet_id - 1]["declined-addresses"] = 1; + + // Now let's add leases to subnet 2. + subnet_id = 2; + + // Insert one delined lease. + makeLease4("192.0.2.2", subnet_id, Lease::STATE_DECLINED); + + // Update the expected stats. + expectedStats[subnet_id - 1]["declined-addresses"] = 1; + + // Now Recount the stats. + ASSERT_NO_THROW(lmptr_->recountAddressStats4()); + + // Make sure stats are as expected. + ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats)); + + // Delete some leases from subnet, and update the expected stats. + EXPECT_TRUE(lmptr_->deleteLease(IOAddress("192.0.1.1"))); + expectedStats[0]["assigned-addresses"] = 1; + + EXPECT_TRUE(lmptr_->deleteLease(IOAddress("192.0.1.2"))); + expectedStats[0]["declined-addresses"] = 0; + + // Recount the stats. + ASSERT_NO_THROW(lmptr_->recountAddressStats4()); + + // Make sure stats are as expected. + ASSERT_NO_FATAL_FAILURE(checkAddressStats4(expectedStats)); +} + }; // namespace test }; // namespace dhcp }; // namespace isc diff --git a/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h b/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h index 3ef1405503..f646412db3 100644 --- a/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h +++ b/src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.h @@ -1,4 +1,4 @@ -// Copyright (C) 2014-2015 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2014-2016 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -15,6 +15,12 @@ namespace isc { namespace dhcp { namespace test { + +/// @brief typedefs to simplify lease statistic testing +typedef std::map StatValMap; +typedef std::pair StatValPair; +typedef std::vector StatValMapList; + /// @brief Test Fixture class with utility functions for LeaseMgr backends /// /// It contains utility functions, like dummy lease creation. @@ -94,6 +100,34 @@ public: /// @return vector Vector of pointers to leases std::vector createLeases6(); + /// @brief Compares a StatsMgr statistic to an expected value + /// + /// Attempt to fetch the named statistic from the StatsMg and if + /// found, compare its observed value to the given value. + /// Fails if the stat is not found or if the values do not match. + /// + /// @param name StatsMgr name for the statistic to check + /// @param expected_value expected value of the statistic + void checkStat(const std::string& name, const int64_t expected_value); + + /// @brief Compares StatsMgr statistics against an expected list of values + /// + /// Iterates over a list of statistic names and expectec values, attempting + /// to fetch each from the StatsMgr and if found, compare its observed value + /// to the expected value. Fails any the stat is not found or if the values + /// do not match. + /// + /// @param expected_stats Map of expected static names and values. + void checkAddressStats4(const StatValMapList& expected_stats); + + /// @brief Constructs a minimal IPv4 lease and adds it to the lease storage + /// + /// @param address - IPv4 address for the lease + /// @param subnet_id - subnet ID to which the lease belongs + /// @param state - the state of the lease + void makeLease4(const std::string& address, const SubnetID& subnet_id, + const Lease::LeaseState& state = Lease::STATE_DEFAULT); + /// @brief checks that addLease, getLease4(addr) and deleteLease() works void testBasicLease4(); @@ -313,6 +347,13 @@ public: /// leases can be removed. void testDeleteExpiredReclaimedLeases4(); + /// @brief Check that the IPv4 lease statistics can be recounted + /// + /// This test creates two subnets and several leases associated with + /// them, then verifies that lease statistics are recalculated correctly + /// after altering the lease states in various ways. + void testRecountAddressStats4(); + /// @brief String forms of IPv4 addresses std::vector straddress4_; diff --git a/src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc b/src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc index a01a54457c..3cf45d58d6 100644 --- a/src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc +++ b/src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc @@ -1883,6 +1883,10 @@ TEST_F(MemfileLeaseMgrTest, lease6ContainerIndexUpdate) { } } - +// Verifies that IPv4 lease statistics can be recalculated. +TEST_F(MemfileLeaseMgrTest, recountAddressStats4) { + startBackend(V4); + testRecountAddressStats4(); +} }; // end of anonymous namespace diff --git a/src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc b/src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc index 33549b415a..bbe80f9b60 100644 --- a/src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc +++ b/src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc @@ -477,4 +477,9 @@ TEST_F(MySqlLeaseMgrTest, deleteExpiredReclaimedLeases4) { testDeleteExpiredReclaimedLeases4(); } +// Verifies that IPv4 lease statistics can be recalculated. +TEST_F(MySqlLeaseMgrTest, recountAddressStats4) { + testRecountAddressStats4(); +} + }; // Of anonymous namespace