From: Marcin Siodelski Date: Tue, 23 Jun 2020 15:24:51 +0000 (+0200) Subject: [#1258] Ported ha-status cmd from 1.7.3 X-Git-Tag: Kea-1.6.3~13 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0f9014f70809056e1142af62b0443db3aa0828be;p=thirdparty%2Fkea.git [#1258] Ported ha-status cmd from 1.7.3 --- diff --git a/src/hooks/dhcp/high_availability/communication_state.cc b/src/hooks/dhcp/high_availability/communication_state.cc index 6ffd48f827..8a1b15ff8d 100644 --- a/src/hooks/dhcp/high_availability/communication_state.cc +++ b/src/hooks/dhcp/high_availability/communication_state.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -22,6 +22,7 @@ #include using namespace isc::asiolink; +using namespace isc::data; using namespace isc::dhcp; using namespace isc::http; using namespace boost::posix_time; @@ -46,8 +47,9 @@ CommunicationState::CommunicationState(const IOServicePtr& io_service, const HAConfigPtr& config) : io_service_(io_service), config_(config), timer_(), interval_(0), poke_time_(boost::posix_time::microsec_clock::universal_time()), - heartbeat_impl_(0), partner_state_(-1), clock_skew_(0, 0, 0, 0), - last_clock_skew_warn_(), my_time_at_skew_(), partner_time_at_skew_() { + heartbeat_impl_(0), partner_state_(-1), partner_scopes_(), + clock_skew_(0, 0, 0, 0), last_clock_skew_warn_(), my_time_at_skew_(), + partner_time_at_skew_() { } CommunicationState::~CommunicationState() { @@ -78,6 +80,28 @@ CommunicationState::setPartnerState(const std::string& state) { } } +void +CommunicationState::setPartnerScopes(ConstElementPtr new_scopes) { + if (!new_scopes || (new_scopes->getType() != Element::list)) { + isc_throw(BadValue, "unable to record partner's HA scopes because" + " the received value is not a valid JSON list"); + } + + std::set partner_scopes; + for (auto i = 0; i < new_scopes->size(); ++i) { + auto scope = new_scopes->get(i); + if (scope->getType() != Element::string) { + isc_throw(BadValue, "unable to record partner's HA scopes because" + " the received scope value is not a valid JSON string"); + } + auto scope_str = scope->stringValue(); + if (!scope_str.empty()) { + partner_scopes.insert(scope_str); + } + } + partner_scopes_ = partner_scopes; +} + void CommunicationState::startHeartbeat(const long interval, const boost::function& heartbeat_impl) { diff --git a/src/hooks/dhcp/high_availability/communication_state.h b/src/hooks/dhcp/high_availability/communication_state.h index 1bc80e889b..69771ecb75 100644 --- a/src/hooks/dhcp/high_availability/communication_state.h +++ b/src/hooks/dhcp/high_availability/communication_state.h @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -100,6 +101,12 @@ public: /// @throw BadValue if unsupported state value was provided. void setPartnerState(const std::string& state); + std::set getPartnerScopes() const { + return (partner_scopes_); + } + + void setPartnerScopes(data::ConstElementPtr new_scopes); + /// @brief Starts recurring heartbeat (public interface). /// /// @param interval heartbeat interval in milliseconds. @@ -313,6 +320,9 @@ protected: /// Negative value means that the partner's state is unknown. int partner_state_; + /// @brief Last known set of scopes served by the partner server. + std::set partner_scopes_; + /// @brief Clock skew between the active servers. boost::posix_time::time_duration clock_skew_; diff --git a/src/hooks/dhcp/high_availability/ha_impl.cc b/src/hooks/dhcp/high_availability/ha_impl.cc index 46fadf68b6..8c1070982a 100644 --- a/src/hooks/dhcp/high_availability/ha_impl.cc +++ b/src/hooks/dhcp/high_availability/ha_impl.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -262,6 +262,23 @@ HAImpl::commandProcessed(hooks::CalloutHandle& callout_handle) { callout_handle.getArgument("name", command_name); if (command_name == "dhcp-enable") { service_->adjustNetworkState(); + } else if (command_name == "status-get") { + // Get the response. + ConstElementPtr response; + callout_handle.getArgument("response", response); + if (!response || (response->getType() != Element::map)) { + return; + } + // Get the arguments item from the response. + ConstElementPtr resp_args = response->get("arguments"); + if (!resp_args || (resp_args->getType() != Element::map)) { + return; + } + // Add the ha servers info to arguments. + ElementPtr mutable_resp_args = + boost::const_pointer_cast(resp_args); + ConstElementPtr ha_servers = service_->processStatusGet(); + mutable_resp_args->set("ha-servers", ha_servers); } } diff --git a/src/hooks/dhcp/high_availability/ha_impl.h b/src/hooks/dhcp/high_availability/ha_impl.h index a27892ba57..5399028e79 100644 --- a/src/hooks/dhcp/high_availability/ha_impl.h +++ b/src/hooks/dhcp/high_availability/ha_impl.h @@ -1,4 +1,4 @@ -// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -118,6 +118,8 @@ public: /// service is enabled in a state for which this is not allowed, e.g. /// waiting, syncing etc. We don't want to rely on the HA partner to do /// a correct thing in that respect. + /// It too adds the HA servers information to "status-get" command + /// responses by calling @c HAService::commandProcessed. /// /// @param callout_handle Callout handle provided to the callout. void commandProcessed(hooks::CalloutHandle& callout_handle); diff --git a/src/hooks/dhcp/high_availability/ha_service.cc b/src/hooks/dhcp/high_availability/ha_service.cc index 77106bef18..36e0d14578 100644 --- a/src/hooks/dhcp/high_availability/ha_service.cc +++ b/src/hooks/dhcp/high_availability/ha_service.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -952,6 +952,75 @@ HAService::logFailedLeaseUpdates(const PktPtr& query, log_proc(query, args, "failed-leases", HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER); } +ConstElementPtr +HAService::processStatusGet() const { + ElementPtr ha_servers = Element::createMap(); + + // Local part + ElementPtr local = Element::createMap(); + HAConfig::PeerConfig::Role role; + role = config_->getThisServerConfig()->getRole(); + std::string role_txt = HAConfig::PeerConfig::roleToString(role); + local->set("role", Element::create(role_txt)); + int state = getCurrState(); + try { + local->set("state", Element::create(stateToString(state))); + + } catch (...) { + // Empty string on error. + local->set("state", Element::create(std::string())); + } + std::set scopes = query_filter_.getServedScopes(); + ElementPtr list = Element::createList(); + for (std::string scope : scopes) { + list->add(Element::create(scope)); + } + local->set("scopes", list); + ha_servers->set("local", local); + + // Remote part + ElementPtr remote = Element::createMap(); + + // Add the in-touch boolean flag to indicate whether there was any + // communication between the HA peers. Based on that, the user + // may determine if the status returned for the peer is based on + // the heartbeat or is to be determined. + auto in_touch = (communication_state_->getPartnerState() > 0); + remote->set("in-touch", Element::create(in_touch)); + + auto age = in_touch ? + static_cast(communication_state_->getDurationInMillisecs() / 1000) : 0; + remote->set("age", Element::create(age)); + + try { + role = config_->getFailoverPeerConfig()->getRole(); + std::string role_txt = HAConfig::PeerConfig::roleToString(role); + remote->set("role", Element::create(role_txt)); + + } catch (...) { + remote->set("role", Element::create(std::string())); + } + + try { + state = getPartnerState(); + remote->set("last-state", Element::create(stateToString(state))); + + } catch (...) { + remote->set("last-state", Element::create(std::string())); + } + + // Remote server's scopes. + scopes = communication_state_->getPartnerScopes(); + list = Element::createList(); + for (auto scope : scopes) { + list->add(Element::create(scope)); + } + remote->set("last-scopes", list); + ha_servers->set("remote", remote); + + return (ha_servers); +} + ConstElementPtr HAService::processHeartbeat() { ElementPtr arguments = Element::createMap(); @@ -961,6 +1030,12 @@ HAService::processHeartbeat() { std::string date_time = HttpDateTime().rfc1123Format(); arguments->set("date-time", Element::create(date_time)); + auto scopes = query_filter_.getServedScopes(); + ElementPtr scopes_list = Element::createList(); + for (auto scope : scopes) { + scopes_list->add(Element::create(scope)); + } + arguments->set("scopes", scopes_list); return (createAnswer(CONTROL_RESULT_SUCCESS, "HA peer status returned.", arguments)); } @@ -1031,6 +1106,19 @@ HAService::asyncSendHeartbeat() { // Note the time returned by the partner to calculate the clock skew. communication_state_->setPartnerTime(date_time->stringValue()); + // Remember the scopes served by the partner. + try { + auto scopes = args->get("scopes"); + communication_state_->setPartnerScopes(scopes); + + } catch (...) { + // We don't want to fail if the scopes are missing because + // this would be incompatible with old HA hook library + // versions. We may make it mandatory one day, but during + // upgrades of existing HA setup it would be a real issue + // if we failed here. + } + } catch (const std::exception& ex) { LOG_WARN(ha_logger, HA_HEARTBEAT_FAILED) .arg(partner_config->getLogLabel()) diff --git a/src/hooks/dhcp/high_availability/ha_service.h b/src/hooks/dhcp/high_availability/ha_service.h index 14486e1a80..8fcd1258d0 100644 --- a/src/hooks/dhcp/high_availability/ha_service.h +++ b/src/hooks/dhcp/high_availability/ha_service.h @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -246,6 +246,12 @@ public: /// queries. void waitingStateHandler(); + /// @brief Returns last known state of the partner. + /// @ref CommunicationState::getPartnerState. + int getPartnerState() const { + return (communication_state_->getPartnerState()); + } + protected: /// @brief Transitions to a desired state and logs it. @@ -485,12 +491,13 @@ public: /// a restart. /// /// The ha-heartbeat command takes no arguments. The response contains - /// a server state and timestamp in the following format: + /// a server state, served scopes and timestamp in the following format: /// /// @code /// { /// "arguments": { /// "date-time": "Thu, 01 Feb 2018 21:18:26 GMT", + /// "scopes": [ "server1" ], /// "state": "waiting" /// }, /// "result": 0, @@ -501,6 +508,12 @@ public: /// @return Pointer to the response to the heartbeat. data::ConstElementPtr processHeartbeat(); + /// @brief Processes status-get command and returns a response. + /// + /// @c HAImpl::commandProcessed calls this to add information about the + /// HA servers status into the status-get response. + data::ConstElementPtr processStatusGet() const; + protected: /// @brief Starts asynchronous heartbeat to a peer. diff --git a/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc index 6213d87afe..bce4d1a23c 100644 --- a/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/communication_state_unittest.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -22,6 +22,7 @@ using namespace isc; using namespace isc::asiolink; +using namespace isc::data; using namespace isc::dhcp; using namespace isc::ha; using namespace isc::ha::test; @@ -102,7 +103,42 @@ TEST_F(CommunicationStateTest, partnerState) { // An attempt to set unsupported value should result in exception. EXPECT_THROW(state_.setPartnerState("unsupported"), BadValue); +} +// Verifies that the partner's scopes are set and retrieved correctly. +TEST_F(CommunicationStateTest, partnerScopes) { + // Initially, the scopes should be empty. + ASSERT_TRUE(state_.getPartnerScopes().empty()); + + // Set new partner scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ \"server1\", \"server2\" ]")) + ); + + // Get them back. + auto returned = state_.getPartnerScopes(); + EXPECT_EQ(2, returned.size()); + EXPECT_EQ(1, returned.count("server1")); + EXPECT_EQ(1, returned.count("server2")); + + // Override the scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ \"server1\" ]")) + ); + returned = state_.getPartnerScopes(); + EXPECT_EQ(1, returned.size()); + EXPECT_EQ(1, returned.count("server1")); + + // Clear the scopes. + ASSERT_NO_THROW( + state_.setPartnerScopes(Element::fromJSON("[ ]")) + ); + returned = state_.getPartnerScopes(); + EXPECT_TRUE(returned.empty()); + + // An attempt to set invalid JSON should fail. + EXPECT_THROW(state_.setPartnerScopes(Element::fromJSON("{ \"not-a-list\": 1 }")), + BadValue); } // Verifies that the object is poked right after construction. diff --git a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc index 36f2ad1f47..c2f3575725 100644 --- a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -533,5 +533,54 @@ TEST_F(HAImplTest, continueHandler) { checkAnswer(response, CONTROL_RESULT_SUCCESS, "HA state machine is not paused."); } +// Tests status-get command processed handler. +TEST_F(HAImplTest, statusGet) { + HAImpl ha_impl; + ASSERT_NO_THROW(ha_impl.configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState(NetworkState::DHCPv4)); + ASSERT_NO_THROW(ha_impl.startService(io_service_, network_state, + HAServerType::DHCPv4)); + + std::string name = "status-get"; + ConstElementPtr response = + Element::fromJSON("{ \"arguments\": { \"pid\": 1 }, \"result\": 0 }"); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + + callout_handle->setArgument("name", name); + callout_handle->setArgument("response", response); + + ASSERT_NO_THROW(ha_impl.commandProcessed(*callout_handle)); + + ConstElementPtr got; + callout_handle->getArgument("response", got); + ASSERT_TRUE(got); + + std::string expected = + "{" + " \"arguments\": {" + " \"ha-servers\": {" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ ]," + " \"state\": \"waiting\"" + " }," + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"," + " \"role\": \"secondary\"" + " }" + " }," + " \"pid\": 1" + " }," + " \"result\": 0" + "}"; + EXPECT_TRUE(isEquivalent(got, Element::fromJSON(expected))); +} + } diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc index 9a228acab6..d3f09599da 100644 --- a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC") +// Copyright (C) 2018-2020 Internet Systems Consortium, Inc. ("ISC") // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -1065,6 +1066,25 @@ TEST_F(HAServiceTest, hotStandbyScopeSelectionThisPrimary) { service.verboseTransition(HA_HOT_STANDBY_ST); service.runModel(HAService::NOP_EVT); + // Check the reported info about servers. + ConstElementPtr ha_servers = service.processStatusGet(); + ASSERT_TRUE(ha_servers); + std::string expected = "{" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ \"server1\" ]," + " \"state\": \"hot-standby\"" + " }, " + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"role\": \"standby\"," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + // Set the test size - 65535 queries. const unsigned queries_num = 65535; for (unsigned i = 0; i < queries_num; ++i) { @@ -1093,6 +1113,26 @@ TEST_F(HAServiceTest, hotStandbyScopeSelectionThisStandby) { // ... and HA service using this configuration. TestHAService service(io_service_, network_state_, config_storage); + // Check the reported info about servers. + ConstElementPtr ha_servers = service.processStatusGet(); + ASSERT_TRUE(ha_servers); + + std::string expected = "{" + " \"local\": {" + " \"role\": \"standby\"," + " \"scopes\": [ ]," + " \"state\": \"waiting\"" + " }, " + " \"remote\": {" + " \"age\": 0," + " \"in-touch\": false," + " \"role\": \"primary\"," + " \"last-scopes\": [ ]," + " \"last-state\": \"\"" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + // Set the test size - 65535 queries. const unsigned queries_num = 65535; for (unsigned i = 0; i < queries_num; ++i) { @@ -1599,7 +1639,8 @@ TEST_F(HAServiceTest, processHeartbeat) { HAConfigParser parser; ASSERT_NO_THROW(parser.parse(config_storage, Element::fromJSON(config_text))); - HAService service(io_service_, network_state_, config_storage); + TestHAService service(io_service_, network_state_, config_storage); + service.query_filter_.serveDefaultScopes(); // Process heartbeat command. ConstElementPtr rsp; @@ -1622,6 +1663,15 @@ TEST_F(HAServiceTest, processHeartbeat) { ASSERT_TRUE(date_time); EXPECT_EQ(Element::string, date_time->getType()); + auto scopes_list = args->get("scopes"); + ASSERT_TRUE(scopes_list); + EXPECT_EQ(Element::list, scopes_list->getType()); + ASSERT_EQ(1, scopes_list->size()); + auto scope = scopes_list->get(0); + ASSERT_TRUE(scope); + EXPECT_EQ(Element::string, scope->getType()); + EXPECT_EQ("server1", scope->stringValue()); + // The response should contain the timestamp in the format specified // in RFC1123. We use the HttpDateTime method to parse this timestamp. HttpDateTime t; @@ -2547,7 +2597,7 @@ public: const TestHttpResponseCreatorFactoryPtr& factory, const std::string& initial_state = "waiting") : listener_(listener), factory_(factory), running_(false), - static_date_time_() { + static_date_time_(), static_scopes_() { transition(initial_state); } @@ -2566,6 +2616,13 @@ public: static_date_time_ = static_date_time; } + /// @brief Sets static scopes to be used in responses. + /// + /// @param scopes Fixed scopes set. + void setScopes(const std::set& scopes) { + static_scopes_ = scopes; + } + /// @brief Enable response to commands required for leases synchronization. /// /// Enables dhcp-disable, dhcp-enable and lease4-get-page commands. The last @@ -2615,6 +2672,13 @@ public: if (!static_date_time_.empty()) { response_arguments->set("date-time", Element::create(static_date_time_)); } + if (!static_scopes_.empty()) { + auto json_scopes = Element::createList(); + for (auto scope : static_scopes_) { + json_scopes->add(Element::create(scope)); + } + response_arguments->set("scopes", json_scopes); + } factory_->getResponseCreator()->setArguments(response_arguments); } @@ -2634,6 +2698,9 @@ private: /// @brief Static date-time value to be returned. std::string static_date_time_; + + /// @brief Static scopes to be reported. + std::set static_scopes_; }; /// @brief Shared pointer to a partner. @@ -3012,7 +3079,9 @@ TEST_F(HAServiceStateMachineTest, waitingParterDownLoadBalancingPartnerDown) { EXPECT_EQ(HA_PARTNER_DOWN_ST, service_->getCurrState()); // Partner shows up and (eventually) transitions to READY state. - HAPartner partner(listener2_, factory2_, "ready"); + HAPartner partner(listener2_, factory2_); + partner.setScopes({ "server1", "server2" }); + partner.transition("ready"); partner.startup(); // PARTNER DOWN state: receive a response from the partner indicating that @@ -3024,6 +3093,42 @@ TEST_F(HAServiceStateMachineTest, waitingParterDownLoadBalancingPartnerDown) { ASSERT_FALSE(isCommunicationInterrupted()); ASSERT_FALSE(isFailureDetected()); + // Check the reported info about servers. + ConstElementPtr ha_servers = service_->processStatusGet(); + ASSERT_TRUE(ha_servers); + + // Hard to know what is the age of the remote data. Therefore, let's simply + // grab it from the response. + ASSERT_EQ(Element::map, ha_servers->getType()); + auto remote = ha_servers->get("remote"); + ASSERT_TRUE(remote); + EXPECT_EQ(Element::map, remote->getType()); + auto age = remote->get("age"); + ASSERT_TRUE(age); + EXPECT_EQ(Element::integer, age->getType()); + auto age_value = age->intValue(); + EXPECT_GE(age_value, 0); + + // Now append it to the whole structure for comparison. + std::ostringstream s; + s << age_value; + + std::string expected = "{" + " \"local\": {" + " \"role\": \"primary\"," + " \"scopes\": [ \"server1\", \"server2\" ], " + " \"state\": \"load-balancing\"" + " }, " + " \"remote\": {" + " \"age\": " + s.str() + "," + " \"in-touch\": true," + " \"role\": \"secondary\"," + " \"last-scopes\": [ \"server1\", \"server2\" ]," + " \"last-state\": \"ready\"" + " }" + "}"; + EXPECT_TRUE(isEquivalent(Element::fromJSON(expected), ha_servers)); + // Crash the partner and see whether our server can return to the partner // down state. partner.setControlResult(CONTROL_RESULT_ERROR);