]> git.ipfire.org Git - thirdparty/kea.git/commitdiff
[#1375] added CB connection recovery unittests
authorRazvan Becheriu <razvan@isc.org>
Sun, 15 Nov 2020 19:28:14 +0000 (21:28 +0200)
committerRazvan Becheriu <razvan@isc.org>
Wed, 9 Dec 2020 17:12:46 +0000 (19:12 +0200)
26 files changed:
src/hooks/dhcp/mysql_cb/mysql_cb_dhcp4.cc
src/hooks/dhcp/mysql_cb/mysql_cb_dhcp6.cc
src/hooks/dhcp/mysql_cb/tests/mysql_cb_dhcp4_unittest.cc
src/hooks/dhcp/mysql_cb/tests/mysql_cb_dhcp6_unittest.cc
src/lib/dhcpsrv/tests/Makefile.am
src/lib/dhcpsrv/tests/alloc_engine4_unittest.cc
src/lib/dhcpsrv/tests/alloc_engine6_unittest.cc
src/lib/dhcpsrv/tests/alloc_engine_expiration_unittest.cc
src/lib/dhcpsrv/tests/alloc_engine_hooks_unittest.cc
src/lib/dhcpsrv/tests/alloc_engine_utils.cc
src/lib/dhcpsrv/tests/cql_host_data_source_unittest.cc
src/lib/dhcpsrv/tests/cql_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/host_mgr_unittest.cc
src/lib/dhcpsrv/tests/lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/mysql_host_data_source_unittest.cc
src/lib/dhcpsrv/tests/mysql_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/pgsql_host_data_source_unittest.cc
src/lib/dhcpsrv/tests/pgsql_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/sanity_checks_unittest.cc
src/lib/dhcpsrv/testutils/Makefile.am
src/lib/dhcpsrv/testutils/test_utils.cc [moved from src/lib/dhcpsrv/tests/test_utils.cc with 100% similarity]
src/lib/dhcpsrv/testutils/test_utils.h [moved from src/lib/dhcpsrv/tests/test_utils.h with 100% similarity]
src/lib/process/config_ctl_info.cc
src/lib/process/config_ctl_info.h

index 693898c663b50de02e8f48cfe1c10cec72af530e..6deb393966952defbb54852771af69a6c242732c 100644 (file)
@@ -2329,7 +2329,7 @@ public:
             // Iterate over the configured DBs and instantiate them.
             for (auto db : config_ctl->getConfigDatabases()) {
                 const std::string& access = db.getAccessString();
-                auto parameters = DatabaseConnection::parse(access);
+                auto parameters = db.getParameters();
                 if (ConfigBackendDHCPv4Mgr::instance().delBackend(parameters["type"], access, true)) {
                     ConfigBackendDHCPv4Mgr::instance().addBackend(db.getAccessString());
                 }
index 0fa4fb4e860b08fac614863b9d05b618fc86c0bc..f59ff7f576f52eb76dc31fad5cc10d47f4855e39 100644 (file)
@@ -2772,7 +2772,7 @@ public:
             // Iterate over the configured DBs and instantiate them.
             for (auto db : config_ctl->getConfigDatabases()) {
                 const std::string& access = db.getAccessString();
-                auto parameters = DatabaseConnection::parse(access);
+                auto parameters = db.getParameters();
                 if (ConfigBackendDHCPv6Mgr::instance().delBackend(parameters["type"], access, true)) {
                     ConfigBackendDHCPv6Mgr::instance().addBackend(db.getAccessString());
                 }
index 43da6cef8c4920ecd7424d811552d011a55ea4c6..5cc98f1a49b57babdc3df66e347407dec2db2e7f 100644 (file)
@@ -7,6 +7,7 @@
 #include <config.h>
 #include <mysql_cb_dhcp4.h>
 #include <mysql_cb_impl.h>
+#include <database/database_connection.h>
 #include <database/db_exceptions.h>
 #include <database/server.h>
 #include <database/testutils/schema.h>
 #include <dhcp/option_int.h>
 #include <dhcp/option_space.h>
 #include <dhcp/option_string.h>
+#include <dhcpsrv/cfgmgr.h>
+#include <dhcpsrv/config_backend_dhcp4_mgr.h>
 #include <dhcpsrv/pool.h>
 #include <dhcpsrv/subnet.h>
 #include <dhcpsrv/testutils/mysql_generic_backend_unittest.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <mysql/testutils/mysql_schema.h>
+#include <testutils/multi_threading_utils.h>
+
 #include <boost/shared_ptr.hpp>
 #include <gtest/gtest.h>
 #include <mysql.h>
 #include <map>
 #include <sstream>
 
+using namespace isc;
 using namespace isc::asiolink;
 using namespace isc::db;
 using namespace isc::db::test;
 using namespace isc::data;
 using namespace isc::dhcp;
 using namespace isc::dhcp::test;
+using namespace isc::process;
+using namespace isc::test;
+namespace ph = std::placeholders;
 
 namespace {
 
@@ -4153,4 +4163,515 @@ TEST_F(MySqlConfigBackendDHCPv4Test, multipleAuditEntries) {
     }
 }
 
+class MySqlConfigBackendDHCPv4DbLostCallbackTest : public ::testing::Test {
+public:
+    MySqlConfigBackendDHCPv4DbLostCallbackTest()
+        : db_lost_callback_called_(0), db_recovered_callback_called_(0),
+          db_failed_callback_called_(0),
+          io_service_(boost::make_shared<isc::asiolink::IOService>()) {
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(io_service_);
+        isc::dhcp::TimerMgr::instance()->setIOService(io_service_);
+        isc::dhcp::CfgMgr::instance().clear();
+    }
+
+    ~MySqlConfigBackendDHCPv4DbLostCallbackTest() {
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(isc::asiolink::IOServicePtr());
+        isc::dhcp::TimerMgr::instance()->unregisterTimers();
+        isc::dhcp::CfgMgr::instance().clear();
+    }
+
+    /// @brief Prepares the class for a test.
+    ///
+    /// Invoked by gtest prior test entry, we create the
+    /// appropriate schema and create a basic DB manager to
+    /// wipe out any prior instance
+    virtual void SetUp() {
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(io_service_);
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        // Ensure we have the proper schema with no transient data.
+        createMySQLSchema();
+        isc::dhcp::CfgMgr::instance().clear();
+        isc::dhcp::MySqlConfigBackendDHCPv4::registerBackendType();
+    }
+
+    /// @brief Pre-text exit clean up
+    ///
+    /// Invoked by gtest upon test exit, we destroy the schema
+    /// we created.
+    virtual void TearDown() {
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(isc::asiolink::IOServicePtr());
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        // If data wipe enabled, delete transient data otherwise destroy the schema
+        destroyMySQLSchema();
+        isc::dhcp::CfgMgr::instance().clear();
+        isc::dhcp::MySqlConfigBackendDHCPv4::unregisterBackendType();
+    }
+
+    /// @brief Method which returns the back end specific connection
+    /// string
+    virtual std::string validConnectString() {
+        return (validMySQLConnectionString());
+    }
+
+    /// @brief Method which returns invalid back end specific connection
+    /// string
+    virtual std::string invalidConnectString() {
+        return (connectionString(MYSQL_VALID_TYPE, INVALID_NAME, VALID_HOST,
+                                 VALID_USER, VALID_PASSWORD));
+    }
+
+    /// @brief Verifies open failures do NOT invoke db lost callback
+    ///
+    /// The db lost callback should only be invoked after successfully
+    /// opening the DB and then subsequently losing it. Failing to
+    /// open should be handled directly by the application layer.
+    void testNoCallbackOnOpenFailure();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifying and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbRecoveredCallback was invoked
+    void testDbLostAndRecoveredCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifying and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbFailedCallback was invoked
+    void testDbLostAndFailedCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifyingLost and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbRecoveredCallback was invoked after two reconnect
+    /// attempts (once failing and second triggered by timer)
+    void testDbLostAndRecoveredAfterTimeoutCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifyingLost and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbFailedCallback was invoked after two reconnect
+    /// attempts (once failing and second triggered by timer)
+    void testDbLostAndFailedAfterTimeoutCallback();
+
+    /// @brief Callback function registered with the CB manager
+    bool db_lost_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_lost_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_lost_callback function
+    uint32_t db_lost_callback_called_;
+
+    /// @brief Callback function registered with the CB manager
+    bool db_recovered_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_recovered_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_recovered_callback function
+    uint32_t db_recovered_callback_called_;
+
+    /// @brief Callback function registered with the CB manager
+    bool db_failed_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_failed_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_failed_callback function
+    uint32_t db_failed_callback_called_;
+
+    /// The IOService object, used for all ASIO operations.
+    isc::asiolink::IOServicePtr io_service_;
+};
+
+void
+MySqlConfigBackendDHCPv4DbLostCallbackTest::testNoCallbackOnOpenFailure() {
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = invalidConnectString();
+
+    // Connect to the CB backend.
+    ASSERT_THROW(ConfigBackendDHCPv4Mgr::instance().addBackend(access), DbOpenError);
+
+    io_service_->poll();
+
+    EXPECT_EQ(0, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv4DbLostCallbackTest::testDbLostAndRecoveredCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv4Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()));
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(1, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv4DbLostCallbackTest::testDbLostAndFailedCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv4Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()));
+
+    access = invalidConnectString();
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(1, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv4DbLostCallbackTest::testDbLostAndRecoveredAfterTimeoutCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    std::string extra = " max-reconnect-tries=2 reconnect-wait-time=1";
+    access += extra;
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv4Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()));
+
+    access = invalidConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+
+    access = validConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    sleep(1);
+
+    io_service_->poll();
+
+    // Our recovered connectivity callback should have been invoked.
+    EXPECT_EQ(2, db_lost_callback_called_);
+    EXPECT_EQ(1, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv4DbLostCallbackTest::testDbLostAndFailedAfterTimeoutCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv4DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    std::string extra = " max-reconnect-tries=2 reconnect-wait-time=1";
+    access += extra;
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv4Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()));
+
+    access = invalidConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv4Mgr::instance().getPool()->getAllServers4(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+
+    sleep(1);
+
+    io_service_->poll();
+
+    // Our recovered connectivity callback should have been invoked.
+    EXPECT_EQ(2, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(1, db_failed_callback_called_);
+}
+
+/// @brief Verifies that db lost callback is not invoked on an open failure
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testNoCallbackOnOpenFailure) {
+    MultiThreadingTest mt(false);
+    testNoCallbackOnOpenFailure();
+}
+
+/// @brief Verifies that db lost callback is not invoked on an open failure
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testNoCallbackOnOpenFailureMultiThreading) {
+    MultiThreadingTest mt(true);
+    testNoCallbackOnOpenFailure();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndRecoveredCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndRecoveredCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndRecoveredCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndRecoveredCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndFailedCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndFailedCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndFailedCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndFailedCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndRecoveredAfterTimeoutCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndRecoveredAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndRecoveredAfterTimeoutCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndRecoveredAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndFailedAfterTimeoutCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndFailedAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv4DbLostCallbackTest, testDbLostAndFailedAfterTimeoutCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndFailedAfterTimeoutCallback();
+}
+
 }
index 1ce871302bab149e8ff8d153c0a6e04e4bd01e35..f07fba3f4f728c06b4b84badd6914111aac31f7f 100644 (file)
@@ -7,6 +7,7 @@
 #include <config.h>
 #include <mysql_cb_dhcp6.h>
 #include <asiolink/addr_utilities.h>
+#include <database/database_connection.h>
 #include <database/db_exceptions.h>
 #include <database/testutils/schema.h>
 #include <dhcp/dhcp6.h>
 #include <dhcp/option_int.h>
 #include <dhcp/option_space.h>
 #include <dhcp/option_string.h>
+#include <dhcpsrv/cfgmgr.h>
+#include <dhcpsrv/config_backend_dhcp6_mgr.h>
 #include <dhcpsrv/pool.h>
 #include <dhcpsrv/subnet.h>
 #include <dhcpsrv/testutils/mysql_generic_backend_unittest.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <mysql/testutils/mysql_schema.h>
+#include <testutils/multi_threading_utils.h>
+
 #include <boost/shared_ptr.hpp>
 #include <gtest/gtest.h>
 #include <mysql.h>
 #include <map>
 #include <sstream>
 
+using namespace isc;
 using namespace isc::asiolink;
 using namespace isc::db;
 using namespace isc::db::test;
 using namespace isc::data;
 using namespace isc::dhcp;
 using namespace isc::dhcp::test;
+using namespace isc::process;
+using namespace isc::test;
+namespace ph = std::placeholders;
 
 namespace {
 
@@ -4328,4 +4338,515 @@ TEST_F(MySqlConfigBackendDHCPv6Test, multipleAuditEntries) {
     }
 }
 
+class MySqlConfigBackendDHCPv6DbLostCallbackTest : public ::testing::Test {
+public:
+    MySqlConfigBackendDHCPv6DbLostCallbackTest()
+        : db_lost_callback_called_(0), db_recovered_callback_called_(0),
+          db_failed_callback_called_(0),
+          io_service_(boost::make_shared<isc::asiolink::IOService>()) {
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(io_service_);
+        isc::dhcp::TimerMgr::instance()->setIOService(io_service_);
+        isc::dhcp::CfgMgr::instance().clear();
+    }
+
+    ~MySqlConfigBackendDHCPv6DbLostCallbackTest() {
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(isc::asiolink::IOServicePtr());
+        isc::dhcp::TimerMgr::instance()->unregisterTimers();
+        isc::dhcp::CfgMgr::instance().clear();
+    }
+
+    /// @brief Prepares the class for a test.
+    ///
+    /// Invoked by gtest prior test entry, we create the
+    /// appropriate schema and create a basic DB manager to
+    /// wipe out any prior instance
+    virtual void SetUp() {
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(io_service_);
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        // Ensure we have the proper schema with no transient data.
+        createMySQLSchema();
+        isc::dhcp::CfgMgr::instance().clear();
+        isc::dhcp::MySqlConfigBackendDHCPv6::registerBackendType();
+    }
+
+    /// @brief Pre-text exit clean up
+    ///
+    /// Invoked by gtest upon test exit, we destroy the schema
+    /// we created.
+    virtual void TearDown() {
+        isc::dhcp::MySqlConfigBackendImpl::setIOService(isc::asiolink::IOServicePtr());
+        isc::db::DatabaseConnection::db_lost_callback_ = 0;
+        isc::db::DatabaseConnection::db_recovered_callback_ = 0;
+        isc::db::DatabaseConnection::db_failed_callback_ = 0;
+        // If data wipe enabled, delete transient data otherwise destroy the schema
+        destroyMySQLSchema();
+        isc::dhcp::CfgMgr::instance().clear();
+        isc::dhcp::MySqlConfigBackendDHCPv6::unregisterBackendType();
+    }
+
+    /// @brief Method which returns the back end specific connection
+    /// string
+    virtual std::string validConnectString() {
+        return (validMySQLConnectionString());
+    }
+
+    /// @brief Method which returns invalid back end specific connection
+    /// string
+    virtual std::string invalidConnectString() {
+        return (connectionString(MYSQL_VALID_TYPE, INVALID_NAME, VALID_HOST,
+                                 VALID_USER, VALID_PASSWORD));
+    }
+
+    /// @brief Verifies open failures do NOT invoke db lost callback
+    ///
+    /// The db lost callback should only be invoked after successfully
+    /// opening the DB and then subsequently losing it. Failing to
+    /// open should be handled directly by the application layer.
+    void testNoCallbackOnOpenFailure();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifying and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbRecoveredCallback was invoked
+    void testDbLostAndRecoveredCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifying and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbFailedCallback was invoked
+    void testDbLostAndFailedCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifyingLost and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbRecoveredCallback was invoked after two reconnect
+    /// attempts (once failing and second triggered by timer)
+    void testDbLostAndRecoveredAfterTimeoutCallback();
+
+    /// @brief Verifies the CB manager's behavior if DB connection is lost
+    ///
+    /// This function creates a CB manager with an back end that
+    /// supports connectivity lost callback (currently only MySQL and
+    /// PostgreSQL currently).  It verifies connectivity by issuing a known
+    /// valid query.  Next it simulates connectivity lost by identifyingLost and
+    /// closing the socket connection to the CB backend.  It then reissues
+    /// the query and verifies that:
+    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The registered DbLostCallback was invoked
+    /// -# The registered DbFailedCallback was invoked after two reconnect
+    /// attempts (once failing and second triggered by timer)
+    void testDbLostAndFailedAfterTimeoutCallback();
+
+    /// @brief Callback function registered with the CB manager
+    bool db_lost_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_lost_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_lost_callback function
+    uint32_t db_lost_callback_called_;
+
+    /// @brief Callback function registered with the CB manager
+    bool db_recovered_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_recovered_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_recovered_callback function
+    uint32_t db_recovered_callback_called_;
+
+    /// @brief Callback function registered with the CB manager
+    bool db_failed_callback(db::ReconnectCtlPtr /* not_used */) {
+        return (++db_failed_callback_called_);
+    }
+
+    /// @brief Flag used to detect calls to db_failed_callback function
+    uint32_t db_failed_callback_called_;
+
+    /// The IOService object, used for all ASIO operations.
+    isc::asiolink::IOServicePtr io_service_;
+};
+
+void
+MySqlConfigBackendDHCPv6DbLostCallbackTest::testNoCallbackOnOpenFailure() {
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = invalidConnectString();
+
+    // Connect to the CB backend.
+    ASSERT_THROW(ConfigBackendDHCPv6Mgr::instance().addBackend(access), DbOpenError);
+
+    io_service_->poll();
+
+    EXPECT_EQ(0, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv6DbLostCallbackTest::testDbLostAndRecoveredCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv6Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()));
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(1, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv6DbLostCallbackTest::testDbLostAndFailedCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv6Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()));
+
+    access = invalidConnectString();
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(1, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv6DbLostCallbackTest::testDbLostAndRecoveredAfterTimeoutCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    std::string extra = " max-reconnect-tries=2 reconnect-wait-time=1";
+    access += extra;
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv6Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()));
+
+    access = invalidConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+
+    access = validConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    sleep(1);
+
+    io_service_->poll();
+
+    // Our recovered connectivity callback should have been invoked.
+    EXPECT_EQ(2, db_lost_callback_called_);
+    EXPECT_EQ(1, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+}
+
+void
+MySqlConfigBackendDHCPv6DbLostCallbackTest::testDbLostAndFailedAfterTimeoutCallback() {
+    // Set the connectivity lost callback.
+    isc::db::DatabaseConnection::db_lost_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_lost_callback, this, ph::_1);
+
+    // Set the connectivity recovered callback.
+    isc::db::DatabaseConnection::db_recovered_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_recovered_callback, this, ph::_1);
+
+    // Set the connectivity failed callback.
+    isc::db::DatabaseConnection::db_failed_callback_ =
+        std::bind(&MySqlConfigBackendDHCPv6DbLostCallbackTest::db_failed_callback, this, ph::_1);
+
+    std::string access = validConnectString();
+    std::string extra = " max-reconnect-tries=2 reconnect-wait-time=1";
+    access += extra;
+    ConfigControlInfoPtr config_ctl_info(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+
+    // Find the most recently opened socket. Our SQL client's socket should
+    // be the next one.
+    int last_open_socket = findLastSocketFd();
+
+    // Fill holes.
+    FillFdHoles holes(last_open_socket);
+
+    // Connect to the CB backend.
+    ASSERT_NO_THROW(ConfigBackendDHCPv6Mgr::instance().addBackend(access));
+
+    // Find the SQL client socket.
+    int sql_socket = findLastSocketFd();
+    ASSERT_TRUE(sql_socket > last_open_socket);
+
+    // Verify we can execute a query.  We don't care about the answer.
+    ServerCollection servers;
+    ASSERT_NO_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()));
+
+    access = invalidConnectString();
+    access += extra;
+    CfgMgr::instance().clear();
+    // by adding an extra space in the access string will cause the DatabaseConnection::parse
+    // to throw resulting in failure to recreate the manager
+    config_ctl_info.reset(new ConfigControlInfo());
+    config_ctl_info->addConfigDatabase(access);
+    CfgMgr::instance().getCurrentCfg()->setConfigControlInfo(config_ctl_info);
+    const ConfigDbInfoList& cfg = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo()->getConfigDatabases();
+    (const_cast<ConfigDbInfoList&>(cfg))[0].setAccessString(access + " ", true);
+
+    // Now close the sql socket out from under backend client
+    ASSERT_EQ(0, close(sql_socket));
+
+    // A query should fail with DbConnectionUnusable.
+    ASSERT_THROW(servers = ConfigBackendDHCPv6Mgr::instance().getPool()->getAllServers6(BackendSelector()),
+                 DbConnectionUnusable);
+
+    io_service_->poll();
+
+    // Our lost and recovered connectivity callback should have been invoked.
+    EXPECT_EQ(1, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(0, db_failed_callback_called_);
+
+    sleep(1);
+
+    io_service_->poll();
+
+    // Our recovered connectivity callback should have been invoked.
+    EXPECT_EQ(2, db_lost_callback_called_);
+    EXPECT_EQ(0, db_recovered_callback_called_);
+    EXPECT_EQ(1, db_failed_callback_called_);
+}
+
+/// @brief Verifies that db lost callback is not invoked on an open failure
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testNoCallbackOnOpenFailure) {
+    MultiThreadingTest mt(false);
+    testNoCallbackOnOpenFailure();
+}
+
+/// @brief Verifies that db lost callback is not invoked on an open failure
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testNoCallbackOnOpenFailureMultiThreading) {
+    MultiThreadingTest mt(true);
+    testNoCallbackOnOpenFailure();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndRecoveredCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndRecoveredCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndRecoveredCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndRecoveredCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndFailedCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndFailedCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndFailedCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndFailedCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndRecoveredAfterTimeoutCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndRecoveredAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndRecoveredAfterTimeoutCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndRecoveredAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndFailedAfterTimeoutCallback) {
+    MultiThreadingTest mt(false);
+    testDbLostAndFailedAfterTimeoutCallback();
+}
+
+/// @brief Verifies that loss of connectivity to MySQL is handled correctly.
+TEST_F(MySqlConfigBackendDHCPv6DbLostCallbackTest, testDbLostAndFailedAfterTimeoutCallbackMultiThreading) {
+    MultiThreadingTest mt(true);
+    testDbLostAndFailedAfterTimeoutCallback();
+}
+
 }
index 8aad886da0fdd6bc4d1e39858781e80c5ad1c0b9..c7d8bf2aa4aad2f66f8050cd3c3ce30d1d47896c 100644 (file)
@@ -131,7 +131,6 @@ libdhcpsrv_unittests_SOURCES += srv_config_unittest.cc
 libdhcpsrv_unittests_SOURCES += subnet_unittest.cc
 libdhcpsrv_unittests_SOURCES += test_get_callout_handle.cc test_get_callout_handle.h
 libdhcpsrv_unittests_SOURCES += triplet_unittest.cc
-libdhcpsrv_unittests_SOURCES += test_utils.cc test_utils.h
 libdhcpsrv_unittests_SOURCES += timer_mgr_unittest.cc
 libdhcpsrv_unittests_SOURCES += network_state_unittest.cc
 libdhcpsrv_unittests_SOURCES += network_unittest.cc
index 6ffd1d54ec775c8988d38d04611ae85f3dd8acb5..ffb2696f626073db55a8cc419c72bfe59b2f3f9a 100644 (file)
@@ -11,7 +11,7 @@
 #include <dhcpsrv/shared_network.h>
 #include <dhcpsrv/host_mgr.h>
 #include <dhcpsrv/tests/alloc_engine_utils.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <testutils/gtest_utils.h>
 #include <hooks/hooks_manager.h>
 #include <hooks/callout_handle.h>
index ff2ab7c656aa1ab308a680eec87f86961ba1999d..b1c9f855afd23fabe6ffb0967eccfb7f7129e60a 100644 (file)
@@ -9,7 +9,7 @@
 #include <dhcp/pkt6.h>
 #include <dhcpsrv/host_mgr.h>
 #include <dhcpsrv/tests/alloc_engine_utils.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <stats/stats_mgr.h>
 #include <testutils/gtest_utils.h>
 
index be89802b7a2fb8ee3f47359792d9bc33aa21edf4..c758e476c02c6bef64a3aa64fc3ba5cf2a3fbf71 100644 (file)
@@ -9,7 +9,7 @@
 #include <dhcp/option_data_types.h>
 #include <dhcp_ddns/ncr_msg.h>
 #include <dhcpsrv/tests/alloc_engine_utils.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <hooks/hooks_manager.h>
 #include <stats/stats_mgr.h>
 #include <gtest/gtest.h>
index 682e64aa4c7a6274d3a0f80a8d483f60d09806a7..935296dcbaa761be8f28972ffee1cc4df598dfdb 100644 (file)
@@ -6,7 +6,7 @@
 
 #include <config.h>
 #include <dhcpsrv/tests/alloc_engine_utils.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 
 #include <hooks/server_hooks.h>
 #include <hooks/callout_manager.h>
index 9de1ea699043788f0736a3f2252012dae593df09..03d224bfcb9cf7aeaf15c20a523205fcd9615eff 100644 (file)
@@ -18,7 +18,7 @@
 #include <hooks/callout_handle.h>
 #include <stats/stats_mgr.h>
 
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/alloc_engine_utils.h>
 
 #include <hooks/hooks_manager.h>
index ec2b80b2ef529dbbe4a0af0a7e7da369c1d68005..17b084c74815e07164e522fa452abf022a2a3315 100644 (file)
@@ -18,7 +18,7 @@
 #include <config.h>
 
 #include <asiolink/io_address.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <exceptions/exceptions.h>
 #include <dhcpsrv/host.h>
 #include <dhcpsrv/cql_host_data_source.h>
index d79dd8bb69afecc744006701d4d4d95cf0ae42ea..38f4455e43c921ac84697cd3fa01f09a95f8a798 100644 (file)
@@ -23,7 +23,7 @@
 #include <cql/testutils/cql_schema.h>
 #include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/cql_lease_mgr.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <exceptions/exceptions.h>
 
index cc8966fdf88d2d7ab3feb32a5a8374cb70b2ae91..1d3524819c0610c997f0f0f76af8049f7f322ed3 100644 (file)
@@ -13,7 +13,7 @@
 #include <dhcpsrv/dhcpsrv_exceptions.h>
 #include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <exceptions/exceptions.h>
 #include <stats/stats_mgr.h>
 
index 014a1b8b18db4b01db1a4fd1eaf5e786e5a3919d..564f863407e29c6b29bfdc71c0f743fe53585b87 100644 (file)
@@ -11,7 +11,7 @@
 #include <dhcpsrv/host.h>
 #include <dhcpsrv/host_data_source_factory.h>
 #include <dhcpsrv/host_mgr.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <testutils/multi_threading_utils.h>
 #include <util/multi_threading_mgr.h>
 
index a430c789c07cc19f6c4a971dfc0cfa35c7c45413..01dadd931ae1ff0001e1e5e5966a3241d7a2eaec 100644 (file)
@@ -9,7 +9,7 @@
 #include <asiolink/io_address.h>
 #include <dhcpsrv/lease_mgr.h>
 #include <dhcpsrv/memfile_lease_mgr.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 
 #include <gtest/gtest.h>
index 25d4a5541bfeabb47a5e340559034107c04ec253..937f8a3b4e809aba9072faa210219e9df9999371 100644 (file)
@@ -16,7 +16,7 @@
 #include <dhcpsrv/memfile_lease_mgr.h>
 #include <dhcpsrv/timer_mgr.h>
 #include <dhcpsrv/testutils/lease_file_io.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <util/multi_threading_mgr.h>
 #include <util/pid_file.h>
index c28417e708b944aa6821adb83f05659401db0153..677a0679bd926d625820497cf4a64815d2a443a7 100644 (file)
@@ -7,7 +7,7 @@
 #include <config.h>
 
 #include <asiolink/io_address.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <exceptions/exceptions.h>
 #include <dhcpsrv/host.h>
 #include <dhcpsrv/mysql_host_data_source.h>
index 82f2185a55b50588d95768ba4a157432c59e1313..d740b644301581e968c6425f502ad3d8f89505d7 100644 (file)
@@ -9,7 +9,7 @@
 #include <asiolink/io_address.h>
 #include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/mysql_lease_mgr.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <exceptions/exceptions.h>
 #include <mysql/mysql_connection.h>
index c7e5546831bc756f672a4bcee409133b4a073482..6f6985037b17f07cb7eadaf52979ca1cb0af5918 100644 (file)
@@ -7,7 +7,7 @@
 #include <config.h>
 
 #include <asiolink/io_address.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <exceptions/exceptions.h>
 #include <dhcpsrv/host.h>
 #include <dhcpsrv/pgsql_host_data_source.h>
index f5433d01bae7b1f5a1b54de98434efaae7df4de4..393c0d36e3461c4e78cd4dd7187e99300e0bd19e 100644 (file)
@@ -9,7 +9,7 @@
 #include <asiolink/io_address.h>
 #include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/pgsql_lease_mgr.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <exceptions/exceptions.h>
 #include <pgsql/pgsql_connection.h>
index a7231e1baa8570f5306efce397748eb28c6bac94..6831d308a2340b34fc7e81687e90a81a482a7678 100644 (file)
@@ -13,7 +13,7 @@
 #include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/subnet.h>
 #include <dhcpsrv/sanity_checker.h>
-#include <dhcpsrv/tests/test_utils.h>
+#include <dhcpsrv/testutils/test_utils.h>
 #include <util/range_utilities.h>
 #include <cc/data.h>
 #include <gtest/gtest.h>
index 18ec74d9fa4bda78acb88b9f4aabef4e41de132e..cef0020f29f185350b1cd2abf92fcb93a3360d4f 100644 (file)
@@ -16,6 +16,7 @@ libdhcpsrvtest_la_SOURCES  = config_result_check.cc config_result_check.h
 libdhcpsrvtest_la_SOURCES += dhcp4o6_test_ipc.cc dhcp4o6_test_ipc.h
 libdhcpsrvtest_la_SOURCES += host_data_source_utils.cc host_data_source_utils.h
 libdhcpsrvtest_la_SOURCES += memory_host_data_source.cc memory_host_data_source.h
+libdhcpsrvtest_la_SOURCES += test_utils.cc test_utils.h
 libdhcpsrvtest_la_SOURCES += generic_backend_unittest.cc generic_backend_unittest.h
 libdhcpsrvtest_la_SOURCES += generic_host_data_source_unittest.cc generic_host_data_source_unittest.h
 libdhcpsrvtest_la_SOURCES += lease_file_io.cc lease_file_io.h
index 33edb91c50795d2064a2ab4f69c9a5d3450c7a6e..7e38994479449ed9a4f1a0cb29daec2ec418fccb 100644 (file)
@@ -14,10 +14,12 @@ namespace isc {
 namespace process {
 
 void
-ConfigDbInfo::setAccessString(const std::string& access_str) {
+ConfigDbInfo::setAccessString(const std::string& access_str, bool test_mode) {
     access_str_ = access_str;
     access_params_.clear();
-    access_params_ = db::DatabaseConnection::parse(access_str_);
+    if (!test_mode) {
+        access_params_ = db::DatabaseConnection::parse(access_str_);
+    }
 }
 
 bool
index 96f961641ad3500e96c11b966879182ecc146e97..8440b3a9ccd2b7fd47f668355d43f80812aaa15a 100644 (file)
@@ -19,34 +19,36 @@ namespace isc {
 namespace process {
 
 /// @brief Provides configuration information used during a server's
-/// configuration process
+/// configuration process.
 ///
 class ConfigDbInfo : public isc::data::CfgToElement {
 public:
     /// @brief Constructor
     ConfigDbInfo() {};
 
-    /// @brief Set the access string
+    /// @brief Set the access string.
     ///
-    /// Sest the db's access string to the given value and then parses it
+    /// Sets the db's access string to the given value and then parses it
     /// into name-value pairs and storing them internally as a
     /// DatabaseConnection::ParameterMap.  It discards the existing content
     /// of the map first.  It does not validate the parameter names of values,
     /// ensuring the validity of the string content is placed upon the caller.
     ///
-    /// @param access_str string of name=value pairs seperated by spaces
-    void setAccessString(const std::string& access_str);
+    /// @param access_str string of name=value pairs separated by spaces.
+    /// @param test_mode flag used in unittests only to skip parsing the access
+    /// string and storing the parameters.
+    void setAccessString(const std::string& access_str, bool test_mode = false);
 
     /// @brief Retrieves the database access string.
     ///
-    /// @return database access string
+    /// @return database access string.
     std::string getAccessString() const {
         return (access_str_);
     }
 
     /// @brief Retrieves the database access string with password redacted.
     ///
-    /// @return database access string with password redacted
+    /// @return database access string with password redacted.
     std::string redactedAccessString() const {
         return(db::DatabaseConnection::redactedAccessString(access_params_));
     }
@@ -58,9 +60,9 @@ public:
         return (access_params_);
     }
 
-    /// @brief Fetch the value of a given parmeter
+    /// @brief Fetch the value of a given parameter.
     ///
-    /// @param name name of the parameter value to fetch
+    /// @param name name of the parameter value to fetch.
     /// @param[out] value string which will contain the value of the
     /// parameter (if found).
     ///
@@ -69,9 +71,9 @@ public:
     bool getParameterValue(const std::string& name,
                            std::string& value) const;
 
-    /// @brief Unparse a configuration object
+    /// @brief Unparse a configuration object.
     ///
-    /// @return a pointer to unparsed configuration
+    /// @return a pointer to unparsed configuration.
     virtual isc::data::ElementPtr toElement() const;
 
     /// @brief Compares two objects for equality.
@@ -100,17 +102,17 @@ public:
     }
 
 private:
-    /// @brief Access string of parameters used to acces this database
+    /// @brief Access string of parameters used to access this database.
     std::string access_str_;
 
-    /// @brief Map of the access parameters and their values
+    /// @brief Map of the access parameters and their values.
     db::DatabaseConnection::ParameterMap access_params_;
 };
 
 typedef std::vector<ConfigDbInfo> ConfigDbInfoList;
 
 /// @brief Embodies configuration information used during a server's
-/// configuration process
+/// configuration process.
 ///
 /// This is class conveys the configuration control information
 /// described by the following JSON text:
@@ -178,24 +180,24 @@ public:
     /// in the given access string, or if the access string is invalid.
     void addConfigDatabase(const std::string& access_str);
 
-    /// @brief Retrieves the list of databases
+    /// @brief Retrieves the list of databases.
     ///
     /// The entries in the list are stored in the order they were
     /// added to it (FIFO).
     ///
-    /// @return a reference to a const list of databases
+    /// @return a reference to a const list of databases.
     const ConfigDbInfoList& getConfigDatabases() const {
         return (db_infos_);
     }
 
-    /// @brief Retrieves the datbase with the given access parameter value
+    /// @brief Retrieves the database with the given access parameter value.
     ///
     /// @return A reference to the matching database or the not-found value
-    /// available via @c EMPTY_DB()
+    /// available via @c EMPTY_DB().
     const ConfigDbInfo& findConfigDb(const std::string& param_name,
                                      const std::string& param_value);
 
-    /// @brief Empties the contents of the class, including the database list
+    /// @brief Empties the contents of the class, including the database list.
     void clear();
 
     /// @brief Merges specified configuration into this configuration.
@@ -207,14 +209,14 @@ public:
     /// configuration.
     void merge(const ConfigControlInfo& other);
 
-    /// @brief Unparse a configuration object
+    /// @brief Unparse a configuration object.
     ///
-    /// @return a pointer to unparsed configuration
+    /// @return a pointer to unparsed configuration.
     virtual isc::data::ElementPtr toElement() const;
 
-    /// @brief Fetches the not-found value returned by database list searches
+    /// @brief Fetches the not-found value returned by database list searches.
     ///
-    /// @return a reference to the empty ConfigDBInfo
+    /// @return a reference to the empty ConfigDBInfo.
     static const ConfigDbInfo& EMPTY_DB();
 
     /// @brief Compares two objects for equality.
@@ -229,13 +231,14 @@ private:
     /// @brief Configured value of the config-fetch-wait-time.
     util::Optional<uint16_t> config_fetch_wait_time_;
 
-    /// @brief List of configuration databases
+    /// @brief List of configuration databases.
     ConfigDbInfoList db_infos_;
 };
 
-/// @brief Defines a pointer to a ConfigControlInfo
+/// @brief Defines a pointer to a ConfigControlInfo.
 typedef boost::shared_ptr<ConfigControlInfo> ConfigControlInfoPtr;
-/// @brief Defines a pointer to a const ConfigControlInfo
+
+/// @brief Defines a pointer to a const ConfigControlInfo.
 typedef boost::shared_ptr<const ConfigControlInfo> ConstConfigControlInfoPtr;
 
 } // namespace process