[#1114] Servers execute shutdown on unrecoverable DBs

    Backprot #1108 changes to v1_6_0.

Added ChangeLog entry

src/bin/dhcp4/ctrl_dhcp4_srv.*
    ControlledDhcpv4Srv::dbLostCallback() - schedules a shutdown
    once retries have been exhausted/disableld

src/bin/dhcp6/ctrl_dhcp6_srv.*
    ControlledDhcpv6Srv::dbLostCallback() - schedules a shutdown
    once retries have been exhausted/disableld

src/lib/database/database_connection.h
    class DbUnrecoverableError - new exception

src/lib/mysql/mysql_connection.h
    MySqlConnection::check_error() - throws DbUnrecoverableError
    instead of calling exit()

src/lib/pgsql/pgsql_connection.*
    PgSqlConnection::checkStatementError() - throws DbUnrecoverableError
    instead of calling exit()

diff --git a/ChangeLog b/ChangeLog
index d4c83456b01fff238951cc80824be5abea37c52c..2190483d8b5de3dcb29497d23a8c0a84a9ba964f 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+1662.  [bug]           tmark
+       kea-dhcp4 and kea-dhcp6 now shutdown gracefully by executing
+       the shutdown command, if connectivity with a backend database
+       has been lost and retries are either disabled or have been
+       exhausted. Prior to this they simply invoked exit() which
+       could orphan control socket files or cause segfaults unloading
+       the CB Cmds hook library.
+       (Gitlab #1114,#1108)
+
 1661.  [bug]           tmark
        Kea servers now detect and remove orphaned control channel
        sockets.  This corrects a failure of the servers to restart

diff --git a/src/bin/dhcp4/ctrl_dhcp4_srv.cc b/src/bin/dhcp4/ctrl_dhcp4_srv.cc
index 4987613534b02ad50388f2f137a2f9660b7c7990..621d155706395de5c9922cc1647979f20c06fcc8 100644 (file)
--- a/src/bin/dhcp4/ctrl_dhcp4_srv.cc
+++ b/src/bin/dhcp4/ctrl_dhcp4_srv.cc
@@ -1,4 +1,4 @@
-// Copyright (C) 2014-2019 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2014-2020 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -1017,12 +1017,14 @@ ControlledDhcpv4Srv::dbLostCallback(ReconnectCtlPtr db_reconnect_ctl) {
         return (false);
     }
 
-    // If reconnect isn't enabled, log it and return false
+    // If reconnect isn't enabled or we're out of retries,
+    // log it, schedule a shutdown,  and return false
     if (!db_reconnect_ctl->retriesLeft() ||
         !db_reconnect_ctl->retryInterval()) {
         LOG_INFO(dhcp4_logger, DHCP4_DB_RECONNECT_DISABLED)
             .arg(db_reconnect_ctl->retriesLeft())
             .arg(db_reconnect_ctl->retryInterval());
+        ControlledDhcpv4Srv::processCommand("shutdown", ConstElementPtr());
         return(false);
     }
 

diff --git a/src/bin/dhcp4/ctrl_dhcp4_srv.h b/src/bin/dhcp4/ctrl_dhcp4_srv.h
index ae8095c2a7d23b8ee7dc70498a9db56fb4314619..8b872a5ae40055521459fc4d4fd4930a9d70900c 100644 (file)
--- a/src/bin/dhcp4/ctrl_dhcp4_srv.h
+++ b/src/bin/dhcp4/ctrl_dhcp4_srv.h
@@ -362,8 +362,8 @@ private:
     /// between retry attempts.
     ///
     /// If either value is zero, reconnect is presumed to be disabled and
-    /// the function will returns false.  This instructs the DB backend
-    /// layer (the caller) to treat the connectivity loss as fatal.
+    /// the function will schedule a shutdown and return false.  This instructs
+    /// the DB backend layer (the caller) to treat the connectivity loss as fatal.
     ///
     /// Otherwise, the function saves db_reconnect_ctl and invokes
     /// dbReconnect to initiate the reconnect process.

diff --git a/src/bin/dhcp6/ctrl_dhcp6_srv.cc b/src/bin/dhcp6/ctrl_dhcp6_srv.cc
index 9e5947606cf24426de9b0ef140d366524d3fc9a0..947410e429e0d39911623677ed2e6edd17133bc5 100644 (file)
--- a/src/bin/dhcp6/ctrl_dhcp6_srv.cc
+++ b/src/bin/dhcp6/ctrl_dhcp6_srv.cc
@@ -1046,6 +1046,7 @@ ControlledDhcpv6Srv::dbLostCallback(ReconnectCtlPtr db_reconnect_ctl) {
         LOG_INFO(dhcp6_logger, DHCP6_DB_RECONNECT_DISABLED)
             .arg(db_reconnect_ctl->retriesLeft())
             .arg(db_reconnect_ctl->retryInterval());
+        ControlledDhcpv6Srv::processCommand("shutdown", ConstElementPtr());
         return(false);
     }
 

diff --git a/src/bin/dhcp6/ctrl_dhcp6_srv.h b/src/bin/dhcp6/ctrl_dhcp6_srv.h
index 83a2d29ab42fe08b9cefb335d8b27abe1bb4a697..3ff7e0c59236c2b247541c4f6d82af7b8dfd0052 100644 (file)
--- a/src/bin/dhcp6/ctrl_dhcp6_srv.h
+++ b/src/bin/dhcp6/ctrl_dhcp6_srv.h
@@ -361,8 +361,8 @@ private:
     /// between retry attempts.
     ///
     /// If either value is zero, reconnect is presumed to be disabled and
-    /// the function will returns false.  This instructs the DB backend
-    /// layer (the caller) to treat the connectivity loss as fatal.
+    /// the function will schedule a shutdown and return false.  This instructs
+    /// the DB backend layer (the caller) to treat the connectivity loss as fatal.
     ///
     /// Otherwise, the function saves db_reconnect_ctl and invokes
     /// dbReconnect to initiate the reconnect process.

diff --git a/src/lib/database/database_connection.h b/src/lib/database/database_connection.h
index 22336b66bd6f2b16554af16fc35391e69d4d6a04..a4164f3341182ef1ac8e2d854524383f9200e3fc 100644 (file)
--- a/src/lib/database/database_connection.h
+++ b/src/lib/database/database_connection.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2015-2019 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2015-2020 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -39,6 +39,14 @@ public:
         isc::Exception(file, line, what) {}
 };
 
+/// @brief Exception thrown when connectivity has been lost and
+/// cannot be recovered.
+class DbUnrecoverableError : public Exception {
+public:
+    DbUnrecoverableError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
 /// @brief Invalid type exception
 ///
 /// Thrown when the factory doesn't recognize the type of the backend.

diff --git a/src/lib/mysql/mysql_connection.h b/src/lib/mysql/mysql_connection.h
index 929fbc16e56db15c69aa97d7c75d3ed83fda3ddf..5e7821d19d982c41912c942854d18972486dbf08 100644 (file)
--- a/src/lib/mysql/mysql_connection.h
+++ b/src/lib/mysql/mysql_connection.h
@@ -536,8 +536,8 @@ public:
     /// If the error is deemed unrecoverable, such as a loss of connectivity
     /// with the server, the function will call invokeDbLostCallback(). If the
     /// invocation returns false then either there is no callback registered
-    /// or the callback has elected not to attempt to reconnect, and exit(-1)
-    /// is called;
+    /// or the callback has elected not to attempt to reconnect, and a
+    /// DbUnrecoverableError is thrown.
     ///
     /// If the invocation returns true, this indicates the calling layer will
     /// attempt recovery, and the function throws a DbOperationError to allow
@@ -574,9 +574,10 @@ public:
                     .arg(mysql_errno(mysql_));
 
                 // If there's no lost db callback or it returns false,
-                // then we're not attempting to recover so we're done
+                // then we're not attempting to recover so we're done.
                 if (!invokeDbLostCallback()) {
-                    exit (-1);
+                    isc_throw(db::DbUnrecoverableError,
+                              "database connectivity cannot be recovered");
                 }
 
                 // We still need to throw so caller can error out of the current

diff --git a/src/lib/pgsql/pgsql_connection.cc b/src/lib/pgsql/pgsql_connection.cc
index af98f098cb1468af6300cdb1743326d979fd1dc6..a77d6ce91c111016a08416a28001e35649a55a66 100644 (file)
--- a/src/lib/pgsql/pgsql_connection.cc
+++ b/src/lib/pgsql/pgsql_connection.cc
@@ -309,9 +309,10 @@ PgSqlConnection::checkStatementError(const PgSqlResult& r,
                 .arg(sqlstate ? sqlstate : "<sqlstate null>");
 
             // If there's no lost db callback or it returns false,
-            // then we're not attempting to recover so we're done
+            // then we're not attempting to recover so we're done.
             if (!invokeDbLostCallback()) {
-                exit (-1);
+                isc_throw(db::DbUnrecoverableError,
+                          "database connectivity cannot be recovered");
             }
 
             // We still need to throw so caller can error out of the current

diff --git a/src/lib/pgsql/pgsql_connection.h b/src/lib/pgsql/pgsql_connection.h
index 63afe01fd062425e19af2f0975669332d0ff3003..69c1576e42f315c264bb321d11e1bbc5972beb2f 100644 (file)
--- a/src/lib/pgsql/pgsql_connection.h
+++ b/src/lib/pgsql/pgsql_connection.h
@@ -386,8 +386,8 @@ public:
     /// If the error is deemed unrecoverable, such as a loss of connectivity
     /// with the server, the function will call invokeDbLostCallback(). If the
     /// invocation returns false then either there is no callback registered
-    /// or the callback has elected not to attempt to reconnect, and exit(-1)
-    /// is called;
+    /// or the callback has elected not to attempt to reconnect, and a
+    /// DbUnrecoverableError is thrown.
     ///
     /// If the invocation returns true, this indicates the calling layer will
     /// attempt recovery, and the function throws a DbOperationError to allow