[#1959] HA state model update

A server in the partner-down state may transition to the waiting state and
synchronize its database when partner informs that it has allocated some
leases for which it did not send lease updates. Otherwise, the server
transitions to the normal state.

diff --git a/src/hooks/dhcp/high_availability/ha_service.cc b/src/hooks/dhcp/high_availability/ha_service.cc
index 5b5a27420ff1d2142a8125f20ae79b4ad3cf9276..e61a91f8270022a4c63922789beab5246279662a 100644 (file)
--- a/src/hooks/dhcp/high_availability/ha_service.cc
+++ b/src/hooks/dhcp/high_availability/ha_service.cc
@@ -505,10 +505,21 @@ HAService::partnerDownStateHandler() {
     case HA_COMMUNICATION_RECOVERY_ST:
     case HA_PARTNER_DOWN_ST:
     case HA_PARTNER_IN_MAINTENANCE_ST:
-    case HA_READY_ST:
         verboseTransition(HA_WAITING_ST);
         break;
 
+    case HA_READY_ST:
+        // If partner allocated new leases for which it didn't send lease updates
+        // to us we should synchronize our database.
+        if (communication_state_->hasPartnerNewUnsentUpdates()) {
+            verboseTransition(HA_WAITING_ST);
+        } else {
+            // We did not miss any lease updates. There is no need to synchronize
+            // the database.
+            verboseTransition(getNormalState());
+        }
+        break;
+
     case HA_TERMINATED_ST:
         verboseTransition(HA_TERMINATED_ST);
         break;
@@ -1683,6 +1694,15 @@ HAService::asyncSendHeartbeat() {
                         // if we failed here.
                     }
 
+                    auto unsent_update_count = args->get("unsent-update-count");
+                    if (unsent_update_count) {
+                        if (unsent_update_count->getType() != Element::integer) {
+                            isc_throw(CtrlChannelError, "unsent-update-count returned in"
+                                      " the ha-heartbeat response is not an integer");
+                        }
+                        communication_state_->setPartnerUnsentUpdateCount(static_cast<uint64_t>(unsent_update_count->intValue()));
+                    }
+
                 } catch (const std::exception& ex) {
                     LOG_WARN(ha_logger, HA_HEARTBEAT_FAILED)
                         .arg(partner_config->getLogLabel())

diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
index 3d485e54ded6d875e21e8674ad3c692e9e914f2f..8b16f19e7910c41fb362893192deae95be761b74 100644 (file)
--- a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
+++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
@@ -483,6 +483,11 @@ private:
             arguments->set("date-time", Element::create(HttpDateTime().rfc1123Format()));
         }
 
+        // Insert unsent-update-count if not present.
+        if (arguments && !arguments->contains("unsent-update-count")) {
+            arguments->set("unsent-update-count", Element::create(int64_t(0)));
+        }
+
         response_body->add(boost::const_pointer_cast<Element>
                            (createAnswer(control_result, "response returned",
                                          arguments)));
@@ -4848,7 +4853,8 @@ public:
               const TestHttpResponseCreatorFactoryPtr& factory,
               const std::string& initial_state = "waiting")
         : listener_(listener), factory_(factory), running_(false),
-          static_date_time_(), static_scopes_() {
+          static_date_time_(), static_scopes_(),
+          static_unsent_update_count_(0) {
         transition(initial_state);
     }
 
@@ -4930,9 +4936,19 @@ public:
             }
             response_arguments->set("scopes", json_scopes);
         }
+        if (static_unsent_update_count_ >= 0) {
+            response_arguments->set("unsent-update-count", Element::create(static_unsent_update_count_));
+        }
         factory_->getResponseCreator()->setArguments(response_arguments);
     }
 
+    /// @brief Sets static value of unsent update count.
+    ///
+    /// @param unsent_update_count new value of the unsent update count.
+    /// Specify a negative value to exclude the count from the response.
+    void setUnsentUpdateCount(int64_t unsent_update_count) {
+        static_unsent_update_count_ = unsent_update_count;
+    }
 
 private:
 
@@ -4952,6 +4968,10 @@ private:
 
     /// @brief Static scopes to be reported.
     std::set<std::string> static_scopes_;
+
+    /// @brief Static count of lease updates not sent by the partner
+    /// because the other server was unavailable.
+    int64_t static_unsent_update_count_;
 };
 
 /// @brief Shared pointer to a partner.
@@ -5371,6 +5391,7 @@ TEST_F(HAServiceStateMachineTest, waitingParterDownLoadBalancingPartnerDown) {
     // Partner shows up and (eventually) transitions to READY state.
     HAPartner partner(listener2_, factory2_);
     partner.setScopes({ "server1", "server2" });
+    partner.setUnsentUpdateCount(10);
     partner.transition("ready");
     partner.startup();
 
@@ -5549,7 +5570,9 @@ TEST_F(HAServiceStateMachineTest, waitingParterDownHotStandbyPartnerDown) {
     EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState());
 
     // Partner shows up and (eventually) transitions to READY state.
-    partner_.reset(new HAPartner(listener_, factory_, "ready"));
+    partner_.reset(new HAPartner(listener_, factory_));
+    partner_->setUnsentUpdateCount(10);
+    partner_->transition("ready");
     partner_->startup();
 
     // PARTNER DOWN state: receive a response from the partner indicating that
@@ -6005,7 +6028,7 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPrimary) {
                        FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
-                       FinalState(HA_WAITING_ST));
+                       FinalState(HA_LOAD_BALANCING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
@@ -6371,7 +6394,7 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingSecondary) {
                        FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
-                       FinalState(HA_WAITING_ST));
+                       FinalState(HA_LOAD_BALANCING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
@@ -6643,7 +6666,7 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsLoadBalancingPause) {
         EXPECT_TRUE(service_->unpause());
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
-                       FinalState(HA_WAITING_ST));
+                       FinalState(HA_LOAD_BALANCING_ST));
         EXPECT_TRUE(state_->isHeartbeatRunning());
     }
 
@@ -7005,7 +7028,7 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyPrimary) {
                        FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
-                       FinalState(HA_WAITING_ST));
+                       FinalState(HA_HOT_STANDBY_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
                        FinalState(HA_PARTNER_DOWN_ST));
@@ -7292,7 +7315,7 @@ TEST_F(HAServiceStateMachineTest, stateTransitionsHotStandbyStandby) {
                        FinalState(HA_WAITING_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_READY_ST),
-                       FinalState(HA_WAITING_ST));
+                       FinalState(HA_HOT_STANDBY_ST));
 
         testTransition(MyState(HA_PARTNER_DOWN_ST), PartnerState(HA_SYNCING_ST),
                        FinalState(HA_PARTNER_DOWN_ST));