]> git.ipfire.org Git - thirdparty/kea.git/commitdiff
[#1396] Implement CB connection recovery
authorThomas Markwalder <tmark@isc.org>
Wed, 5 Aug 2020 20:32:01 +0000 (16:32 -0400)
committerThomas Markwalder <tmark@isc.org>
Wed, 12 Aug 2020 19:19:36 +0000 (15:19 -0400)
src/bin/dhcp4/ctrl_dhcp4_srv.cc
    ControlledDhcpv4Srv::dbReconnect() - added logic to
    reopen CB backends (if any configured)

src/lib/database/database_connection.*
    Added DbConnectionUnusable exception
    DatabaseConnection - added:
        unusable_
        marUnusable()
        checkUnusable()

src/lib/mysql/mysql_connection.*
    MySqlConnection::checkError() - calls markUnusable()

    Added invocations of DatabaseConnection::checkUnusable()
    to functions using the connection

src/lib/pgsql/pgsql_connection.cc
    PgSqlConnection::checkStatementError() - calls markUnusable()

src/bin/dhcp4/ctrl_dhcp4_srv.cc
src/lib/database/database_connection.h
src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc
src/lib/dhcpsrv/tests/host_mgr_unittest.cc
src/lib/mysql/mysql_connection.cc
src/lib/mysql/mysql_connection.h
src/lib/pgsql/pgsql_connection.cc

index b1c20aa577df6b3b10f59fd216fb572a8f726d5c..16e48d87051adb8eb1292d74586d68ce615c835f 100644 (file)
@@ -1136,10 +1136,17 @@ void
 ControlledDhcpv4Srv::dbReconnect(ReconnectCtlPtr db_reconnect_ctl) {
     bool reopened = false;
 
-    // Re-open lease and host database with new parameters.
+    // We lost at least one of them, Reopen all of them (lease, host, and CB databases)
     try {
         CfgDbAccessPtr cfg_db = CfgMgr::instance().getCurrentCfg()->getCfgDbAccess();
         cfg_db->createManagers();
+
+        auto ctl_info = CfgMgr::instance().getCurrentCfg()->getConfigControlInfo();
+        if (ctl_info) {
+            auto srv_cfg = CfgMgr::instance().getCurrentCfg();
+            server_->getCBControl()->databaseConfigConnect(srv_cfg);
+        }
+
         reopened = true;
     } catch (const std::exception& ex) {
         LOG_ERROR(dhcp4_logger, DHCP4_DB_RECONNECT_ATTEMPT_FAILED).arg(ex.what());
@@ -1221,11 +1228,9 @@ ControlledDhcpv4Srv::cbFetchUpdates(const SrvConfigPtr& srv_cfg,
         server_->getCBControl()->databaseConfigFetch(srv_cfg,
                                                      CBControlDHCPv4::FetchMode::FETCH_UPDATE);
         (*failure_count) = 0;
-
     } catch (const std::exception& ex) {
         LOG_ERROR(dhcp4_logger, DHCP4_CB_PERIODIC_FETCH_UPDATES_FAIL)
             .arg(ex.what());
-
         // We allow at most 10 consecutive failures after which we stop
         // making further attempts to fetch the configuration updates.
         // Let's return without re-scheduling the timer.
index a4164f3341182ef1ac8e2d854524383f9200e3fc..773554d716e220f954b5e5035f5579b7db4bb62d 100644 (file)
@@ -47,6 +47,15 @@ public:
         isc::Exception(file, line, what) {}
 };
 
+/// @brief Exception thrown when a specific connection has been rendered unusable
+/// either through loss of connectivity or API lib error
+class DbConnectionUnusable: public Exception {
+public:
+    DbConnectionUnusable(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+
 /// @brief Invalid type exception
 ///
 /// Thrown when the factory doesn't recognize the type of the backend.
@@ -164,7 +173,7 @@ public:
     /// @param parameters A data structure relating keywords and values
     ///        concerned with the database.
     DatabaseConnection(const ParameterMap& parameters)
-        :parameters_(parameters) {
+        :parameters_(parameters), unusable_(false) {
     }
 
     /// @brief Destructor
@@ -240,6 +249,18 @@ public:
     /// open connection subsequently fails
     static DbLostCallback db_lost_callback;
 
+    /// @brief Throws an exception if the connection is not usable.
+    void checkUnusable() {
+        if (unusable_)  {
+            isc_throw (DbConnectionUnusable, "Attempt to use an invalid connection");
+        }
+    }
+
+protected:
+    /// @brief Sets the usable flag to the given value.
+    /// @param usable new value for the flag.
+    void markUnusable() { unusable_ = true; }
+
 private:
 
     /// @brief List of parameters passed in dbconfig
@@ -249,6 +270,13 @@ private:
     /// intended to keep any DHCP-related parameters.
     ParameterMap parameters_;
 
+    /// @brief Indicates if the connection can no longer be used for normal
+    /// operations. Typically a connection is marked unusable after an unrecoverable
+    /// DB error.  There may be a time during which the connection exists after
+    /// such an every during which it cannot be used for anything beyond checking
+    /// parameters and error information.  This flag can be used as a guard in
+    /// code to prevent inadvertant use of a broken connection.
+    bool unusable_;
 };
 
 }  // namespace db
index f8ce430e44d9c581d9bdc28fd69afed7a859f624..d5228e07b304329c82689785bd6c81a4e6add68c 100644 (file)
@@ -3319,9 +3319,9 @@ LeaseMgrDbLostCallbackTest::testDbLostCallback() {
     // Now close the sql socket out from under backend client
     ASSERT_EQ(0, close(sql_socket));
 
-    // A query should fail with DbOperationError.
+    // A query should fail with DbConnectionUnusable.
     ASSERT_THROW(lease = lm.getLease4(IOAddress("192.0.1.0")),
-                 DbOperationError);
+                 DbConnectionUnusable);
 
     // Our lost connectivity callback should have been invoked.
     EXPECT_TRUE(callback_called_);
index c14de8a73d7b24ea3e48d6d66bdb5d57913186ea..6798af604f272b054a28fa3ec6122014737e7838 100644 (file)
@@ -1,4 +1,4 @@
-// Copyright (C) 2014-2019 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2014-2020 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -1109,7 +1109,7 @@ public:
     /// valid query.  Next it simulates connectivity lost by identifying and
     /// closing the socket connection to the host backend.  It then reissues
     /// the query and verifies that:
-    /// -# The Query throws  DbOperationError (rather than exiting)
+    /// -# The Query throws  DbConnectionUnusable (rather than exiting)
     /// -# The registered DbLostCallback was invoked
     void testDbLostCallback();
 #endif
@@ -1154,9 +1154,9 @@ HostMgrDbLostCallbackTest::testDbLostCallback() {
     // Now close the sql socket out from under backend client
     ASSERT_FALSE(close(sql_socket)) << "failed to close socket";
 
-    // A query should fail with DbOperationError.
+    // A query should fail with DbConnectionUnusable.
     ASSERT_THROW(hosts = HostMgr::instance().getAll4(IOAddress("192.0.2.5")),
-                 DbOperationError);
+                 DbConnectionUnusable);
 
     // Our lost connectivity callback should have been invoked.
     EXPECT_TRUE(callback_called_);
index 5e559e36ba4aa16a8d61c752d842b87ccf9505df..b8577c97da452be40ed4cc4bf76608fe39f9ab0f 100644 (file)
@@ -299,6 +299,7 @@ MySqlConnection::getVersion(const ParameterMap& parameters) {
 
 void
 MySqlConnection::prepareStatement(uint32_t index, const char* text) {
+    checkUnusable();
     // Validate that there is space for the statement in the statements array
     // and that nothing has been placed there before.
     if ((index >= statements_.size()) || (statements_[index] != NULL)) {
@@ -325,6 +326,7 @@ MySqlConnection::prepareStatement(uint32_t index, const char* text) {
 void
 MySqlConnection::prepareStatements(const TaggedStatement* start_statement,
                                    const TaggedStatement* end_statement) {
+    checkUnusable();
     // Created the MySQL prepared statements for each DML statement.
     for (const TaggedStatement* tagged_statement = start_statement;
          tagged_statement != end_statement; ++tagged_statement) {
@@ -390,6 +392,7 @@ MySqlConnection::convertFromDatabaseTime(const MYSQL_TIME& expire,
 void
 MySqlConnection::startTransaction() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_START_TRANSACTION);
+    checkUnusable();
     // We create prepared statements for all other queries, but MySQL
     // don't support prepared statements for START TRANSACTION.
     int status = mysql_query(mysql_, "START TRANSACTION");
@@ -402,6 +405,7 @@ MySqlConnection::startTransaction() {
 void
 MySqlConnection::commit() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_COMMIT);
+    checkUnusable();
     if (mysql_commit(mysql_) != 0) {
         isc_throw(DbOperationError, "commit failed: "
                   << mysql_error(mysql_));
@@ -411,6 +415,7 @@ MySqlConnection::commit() {
 void
 MySqlConnection::rollback() {
     DB_LOG_DEBUG(DB_DBG_TRACE_DETAIL, MYSQL_ROLLBACK);
+    checkUnusable();
     if (mysql_rollback(mysql_) != 0) {
         isc_throw(DbOperationError, "rollback failed: "
                   << mysql_error(mysql_));
index 6fbfabef7e67d914144721f676a6d4a323a7ebd4..c9808b803d034ae56b0571f15aa1ef1a6a9b72a4 100644 (file)
@@ -399,6 +399,7 @@ public:
                      const MySqlBindingCollection& in_bindings,
                      MySqlBindingCollection& out_bindings,
                      ConsumeResultFun process_result) {
+        checkUnusable();
         // Extract native input bindings.
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
@@ -476,6 +477,7 @@ public:
     template<typename StatementIndex>
     void insertQuery(const StatementIndex& index,
                      const MySqlBindingCollection& in_bindings) {
+        checkUnusable();
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
             in_bind_vec.push_back(in_binding->getMySqlBinding());
@@ -519,6 +521,7 @@ public:
     template<typename StatementIndex>
     uint64_t updateDeleteQuery(const StatementIndex& index,
                                const MySqlBindingCollection& in_bindings) {
+        checkUnusable();
         std::vector<MYSQL_BIND> in_bind_vec;
         for (MySqlBindingPtr in_binding : in_bindings) {
             in_bind_vec.push_back(in_binding->getMySqlBinding());
@@ -603,7 +606,7 @@ public:
     ///        failed.
     template<typename StatementIndex>
     void checkError(const int status, const StatementIndex& index,
-                    const char* what) const {
+                    const char* what) {
         if (status != 0) {
             switch(mysql_errno(mysql_)) {
                 // These are the ones we consider fatal. Remember this method is
@@ -622,6 +625,9 @@ public:
                     .arg(mysql_error(mysql_))
                     .arg(mysql_errno(mysql_));
 
+                // Mark the connection as no longer usuable.
+                markUnusable();
+
                 // If there's no lost db callback or it returns false,
                 // then we're not attempting to recover so we're done.
                 if (!invokeDbLostCallback()) {
@@ -631,7 +637,7 @@ public:
 
                 // We still need to throw so caller can error out of the current
                 // processing.
-                isc_throw(db::DbOperationError,
+                isc_throw(db::DbConnectionUnusable,
                           "fatal database errror or connectivity lost");
             default:
                 // Connection is ok, so it must be an SQL error
index 0f22511db100bd79745acdc0a344a55f826f9616..7af6be231378ea98c76471c09b3c24b04302e50e 100644 (file)
@@ -335,6 +335,9 @@ PgSqlConnection::checkStatementError(const PgSqlResult& r,
                 .arg(PQerrorMessage(conn_))
                 .arg(sqlstate ? sqlstate : "<sqlstate null>");
 
+            // Mark this connection as no longer usable.
+            markUnusable();
+
             // If there's no lost db callback or it returns false,
             // then we're not attempting to recover so we're done.
             if (!invokeDbLostCallback()) {
@@ -344,7 +347,7 @@ PgSqlConnection::checkStatementError(const PgSqlResult& r,
 
             // We still need to throw so caller can error out of the current
             // processing.
-            isc_throw(DbOperationError,
+            isc_throw(DbConnectionUnusable,
                       "fatal database error or connectivity lost");
         }