[#1402] Watch for invalid partner state

If the local server detects that its partner is in an unexpected state
it will transition to the waiting state. Insufficient checks were indicated
in the review comments.

diff --git a/src/hooks/dhcp/high_availability/ha_messages.cc b/src/hooks/dhcp/high_availability/ha_messages.cc
index 189a97648170bb2afb4249acda0e4282f5e0cbdb..c32a2329f21e60d89a8463836a38c567e4033d28 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_messages.cc
+++ b/src/hooks/dhcp/high_availability/ha_messages.cc
@@ -1,4 +1,4 @@
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Tue Dec 22 2020 09:50
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes
 
 #include <cstddef>
 #include <log/message_types.h>
@@ -43,6 +43,9 @@ extern const isc::log::MessageID HA_HEARTBEAT_HANDLER_FAILED = "HA_HEARTBEAT_HAN
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW = "HA_HIGH_CLOCK_SKEW";
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION = "HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION";
 extern const isc::log::MessageID HA_INIT_OK = "HA_INIT_OK";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY = "HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY = "HA_INVALID_PARTNER_STATE_HOT_STANDBY";
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING = "HA_INVALID_PARTNER_STATE_LOAD_BALANCING";
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_FAILED = "HA_LEASES4_COMMITTED_FAILED";
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE = "HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE";
 extern const isc::log::MessageID HA_LEASES6_COMMITTED_FAILED = "HA_LEASES6_COMMITTED_FAILED";
@@ -134,6 +137,9 @@ const char* values[] = {
     "HA_HIGH_CLOCK_SKEW", "%1, please synchronize clocks!",
     "HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION", "%1, causing HA service to terminate",
     "HA_INIT_OK", "loading High Availability hooks library successful",
+    "HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY", "partner is in the communication-recovery state unexpectedly",
+    "HA_INVALID_PARTNER_STATE_HOT_STANDBY", "partner is in the hot-standby state unexpectedly",
+    "HA_INVALID_PARTNER_STATE_LOAD_BALANCING", "partner is in the load-balancing state unexpectedly",
     "HA_LEASES4_COMMITTED_FAILED", "leases4_committed callout failed: %1",
     "HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE", "%1: leases4_committed callout was invoked without any leases",
     "HA_LEASES6_COMMITTED_FAILED", "leases6_committed callout failed: %1",

diff --git a/src/hooks/dhcp/high_availability/ha_messages.h b/src/hooks/dhcp/high_availability/ha_messages.h
index 76797a0ce1056217a36852fb0eac981495ee32bd..0e72ebb13a145368f967cc7461fd67171646193d 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_messages.h
+++ b/src/hooks/dhcp/high_availability/ha_messages.h
@@ -1,4 +1,4 @@
-// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes on Tue Dec 22 2020 09:50
+// File created from ../../../../src/hooks/dhcp/high_availability/ha_messages.mes
 
 #ifndef HA_MESSAGES_H
 #define HA_MESSAGES_H
@@ -44,6 +44,9 @@ extern const isc::log::MessageID HA_HEARTBEAT_HANDLER_FAILED;
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW;
 extern const isc::log::MessageID HA_HIGH_CLOCK_SKEW_CAUSES_TERMINATION;
 extern const isc::log::MessageID HA_INIT_OK;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY;
+extern const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING;
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_FAILED;
 extern const isc::log::MessageID HA_LEASES4_COMMITTED_NOTHING_TO_UPDATE;
 extern const isc::log::MessageID HA_LEASES6_COMMITTED_FAILED;

diff --git a/src/hooks/dhcp/high_availability/ha_messages.mes b/src/hooks/dhcp/high_availability/ha_messages.mes
index 7dbf11e98d2da22f5b42de5702ae973de3047236..9b2e3f6d2afc8646a8a785793637c7e9d21b26af 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_messages.mes
+++ b/src/hooks/dhcp/high_availability/ha_messages.mes
@@ -228,6 +228,24 @@ servers to resume the HA service.
 This informational message indicates that the High Availability hooks library
 has been loaded successfully.
 
+% HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY partner is in the communication-recovery state unexpectedly
+This warning message is issued when a partner is in the communication-recovery
+state, and this server is not running in the load balancing mode. The server
+may only transition to the communication-recovery state when it runs in the
+load balancing mode. The HA mode of both servers must be the same.
+
+% HA_INVALID_PARTNER_STATE_HOT_STANDBY partner is in the hot-standby state unexpectedly
+This warning message is issued when a partner is in the hot-standby state,
+and this server is not running in the hot standby mode. The server may only
+transition to the hot-standby state when it runs in the hot standby mode.
+The HA mode of both servers must be the same.
+
+% HA_INVALID_PARTNER_STATE_LOAD_BALANCING partner is in the load-balancing state unexpectedly
+This warning message is issued when a partner is in the load-balancing state,
+and this server is not running in the load balancing mode. The server may only
+transition to the load-balancing state when it runs in the load balancing mode.
+The HA mode of both servers must be the same.
+
 % HA_LEASES4_COMMITTED_FAILED leases4_committed callout failed: %1
 This error message is issued when the callout for the leases4_committed hook
 point failed. This includes unexpected errors like wrong arguments provided to

diff --git a/src/hooks/dhcp/high_availability/ha_service.cc b/src/hooks/dhcp/high_availability/ha_service.cc
index 20bcc2f65b3b01529906b94fa83160108e8297a0..cd8ca98bbbdb2e9c2aba350bc57b5ba7f8ec13f8 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_service.cc
+++ b/src/hooks/dhcp/high_availability/ha_service.cc
@@ -184,6 +184,9 @@ HAService::communicationRecoveryHandler() {
     } else if (shouldTerminate()) {
         verboseTransition(HA_TERMINATED_ST);
 
+    } else if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+
     } else {
 
         // Transitions based on the partner's state.
@@ -286,6 +289,14 @@ HAService::normalStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_IN_MAINTENANCE_ST:
         verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
@@ -397,6 +408,14 @@ HAService::partnerDownStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_HOT_STANDBY_ST:
     case HA_LOAD_BALANCING_ST:
@@ -504,6 +523,14 @@ HAService::readyStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_HOT_STANDBY_ST:
     case HA_LOAD_BALANCING_ST:
@@ -573,6 +600,14 @@ HAService::syncingStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's transition to the waiting state and stay there
+    // until the configuration is corrected.
+    if (isPartnerStateInvalid()) {
+        verboseTransition(HA_WAITING_ST);
+        return;
+    }
+
     // We don't want to perform synchronous attempt to synchronize with
     // a partner until we know that the partner is responding. Therefore,
     // we wait for the heartbeat to complete successfully before we
@@ -695,6 +730,14 @@ HAService::waitingStateHandler() {
         return;
     }
 
+    // Check if the partner state is valid per current configuration. If it is
+    // in an invalid state let's sit in the waiting state until the configuration
+    // is corrected.
+    if (isPartnerStateInvalid()) {
+        postNextEvent(NOP_EVT);
+        return;
+    }
+
     switch (communication_state_->getPartnerState()) {
     case HA_COMMUNICATION_RECOVERY_ST:
     case HA_HOT_STANDBY_ST:
@@ -968,6 +1011,36 @@ HAService::isMaintenanceCanceled() const {
     return (getLastEvent() == HA_MAINTENANCE_CANCEL_EVT);
 }
 
+bool
+HAService::isPartnerStateInvalid() const {
+    switch (communication_state_->getPartnerState()) {
+        case HA_COMMUNICATION_RECOVERY_ST:
+            if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY);
+                return (true);
+            }
+            break;
+
+        case HA_HOT_STANDBY_ST:
+            if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_HOT_STANDBY);
+                return (true);
+            }
+            break;
+
+        case HA_LOAD_BALANCING_ST:
+            if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
+                LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_LOAD_BALANCING);
+                return (true);
+            }
+            break;
+
+       default:
+           ;
+    }
+    return (false);
+}
+
 size_t
 HAService::asyncSendLeaseUpdates(const dhcp::Pkt4Ptr& query,
                                  const dhcp::Lease4CollectionPtr& leases,

diff --git a/src/hooks/dhcp/high_availability/ha_service.h b/src/hooks/dhcp/high_availability/ha_service.h
index ebea4f80b8523caf2aa031a0430eb43f92eb1b79..c9aebafc317430fa4a7f9f46fb4201fdcd18e70b 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_service.h
+++ b/src/hooks/dhcp/high_availability/ha_service.h
@@ -470,6 +470,23 @@ protected:
     /// @return true if the maintenance was canceled, false otherwise.
     bool isMaintenanceCanceled() const;
 
+    /// @brief Indicates if the partner's state is invalid.
+    ///
+    /// Partner's state is invalid from the local server's perspective when the
+    /// remote server can't transition to this state if the configuration is
+    /// consistent with the local server's configuration.
+    ///
+    /// The following cases are currently checked:
+    /// - partner in communication-recovery state but this server not in the
+    ///   load balancing mode,
+    /// - partner in the hot-standby state but this server not in the hot standby
+    ///   mode,
+    /// - partner in the load-balancing state but this server not in the
+    ///   load balancing mode.
+    ///
+    /// @return true if the partner's state is invalid, false otherwise.
+    bool isPartnerStateInvalid() const;
+
 public:
 
     /// @brief Schedules asynchronous IPv4 leases updates.

diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
index 5ccc8ab55d9c325bb0f16b1502fdb6258a7bb285..d5dfaf22235ae051eaa8361b03683ad7f1e33e35 100644 (file)
--- a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
+++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
@@ -5376,6 +5376,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5411,6 +5414,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5446,6 +5452,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -5481,6 +5490,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_WAITING_ST));
 
@@ -5516,6 +5528,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -5551,6 +5566,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5586,6 +5604,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -5721,6 +5742,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_COMMUNICATION_RECOVERY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5756,6 +5780,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_LOAD_BALANCING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5791,6 +5818,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -5826,6 +5856,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_WAITING_ST));
 
@@ -5861,6 +5894,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -5896,6 +5932,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_LOAD_BALANCING_ST));
 
@@ -5931,6 +5970,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6373,11 +6415,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("HOT STANDBY state transitions");
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6410,6 +6455,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -6448,6 +6496,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
 
@@ -6480,6 +6531,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_PARTNER_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6513,11 +6567,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("READY state transitions");
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6548,11 +6605,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
         SCOPED_TRACE("WAITING state transitions");
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_SYNCING_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6593,7 +6653,7 @@ TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyPrimary) {
 
     startService(valid_config);
 
-    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                    FinalState(HA_READY_ST));
 
     testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),
@@ -6642,11 +6702,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("HOT STANDBY state transitions");
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_HOT_STANDBY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6679,6 +6742,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
+        testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_HOT_STANDBY_ST),
+                       FinalState(HA_IN_MAINTENANCE_ST));
+
         testTransition(MyState(HA_IN_MAINTENANCE_ST), PartnerState(HA_LOAD_BALANCING_ST),
                        FinalState(HA_IN_MAINTENANCE_ST));
 
@@ -6717,6 +6783,9 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_WAITING_ST));
 
+        testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
 
@@ -6748,11 +6817,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("READY state transitions");
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_HOT_STANDBY_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_READY_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_HOT_STANDBY_ST));
 
+        testTransition(MyState(HA_READY_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_READY_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_PARTNER_IN_MAINTENANCE_ST));
 
@@ -6783,11 +6855,14 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
         SCOPED_TRACE("WAITING state transitions");
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_COMMUNICATION_RECOVERY_ST),
-                       FinalState(HA_SYNCING_ST));
+                       FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                        FinalState(HA_SYNCING_ST));
 
+        testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+                       FinalState(HA_WAITING_ST));
+
         testTransition(MyState(HA_WAITING_ST), PartnerState(HA_IN_MAINTENANCE_ST),
                        FinalState(HA_SYNCING_ST));
 
@@ -6830,7 +6905,7 @@ TEST_F(HAServiceStateMachineTest, noSyncingTransitionsHotStandbyStandby) {
 
     startService(valid_config);
 
-    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_LOAD_BALANCING_ST),
+    testTransition(MyState(HA_WAITING_ST), PartnerState(HA_HOT_STANDBY_ST),
                    FinalState(HA_READY_ST));
 
     testTransition(MyState(HA_WAITING_ST), PartnerState(HA_PARTNER_DOWN_ST),